diff --git a/.clang-tidy b/.clang-tidy index 55411760..44850cdf 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,3 +1,9 @@ +# misc-use-internal-linkage is disabled below: the runtime's foo.cpp + +# foo_internal.hpp split shares helpers across a few TUs through named *_detail +# namespaces. Analyzing one TU the check can't see the cross-TU use and tells us +# to make those `static` (which breaks the link); the genuinely TU-local helpers +# interleaved in the same blocks would each need call-site requalification to +# move to an anonymous namespace. Style only (inlining / symbol-table size). Checks: > -*, bugprone-*, @@ -26,7 +32,8 @@ Checks: > -cppcoreguidelines-owning-memory, -bugprone-easily-swappable-parameters, -cppcoreguidelines-pro-type-union-access, - -cppcoreguidelines-pro-type-vararg + -cppcoreguidelines-pro-type-vararg, + -misc-use-internal-linkage CheckOptions: # libstdc++ implementation headers are not portable include providers. # Prefer the public standard-library header for every standard declaration. diff --git a/README.md b/README.md index 49c578f3..d93a809d 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,8 @@ trades[update { (delta, gamma) = compute_greeks(price) }] User-defined functions can already require a minimum input table schema with `DataFrame<{ ... }>` parameter types. Declared columns must exist with the -right types; extra columns are allowed: +right types; extra columns are allowed and pass through, but cannot be named in +the function body unless the parameter schema declares them: ``` fn top_two_salaries(df: DataFrame<{ salary: Int64 }>) -> DataFrame effects {} { @@ -845,6 +846,7 @@ directory. :schema Show column names and types :head
[n] Show first n rows (default 10) :peek Evaluate and compactly display an expression +:explain Show the physical-plan capability without executing it :describe
[n] Schema + first n rows :doc Show docs/signature for a binding or built-in ?name Shorthand for :doc diff --git a/SPEC.md b/SPEC.md index b9e7070d..0e0fef5b 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3152,6 +3152,11 @@ than an exact-match type. - `DataFrame<{ departmentId: Int, salary: Int }>` requires both columns; other columns remain permitted. +Only the declared fields are statically nameable inside the function. Permitted +extra columns pass through unchanged, but the function cannot reference one by +name unless it adds that column to its parameter schema. The open remainder is +a validation allowance, not a source of dynamically discoverable column names. + This contract is checked at **call time**. A missing required column, or a required column with the wrong type, is a call-time error that names the parameter and the offending column. The same contract applies to a function's diff --git a/include/ibex/ir/join_output.hpp b/include/ibex/ir/join_output.hpp index d8c46f3c..dbc70767 100644 --- a/include/ibex/ir/join_output.hpp +++ b/include/ibex/ir/join_output.hpp @@ -48,6 +48,33 @@ struct JoinOutputColumn { auto operator==(const JoinOutputColumn&) const -> bool = default; }; +/// One equijoin key resolved against the ordered physical columns of both +/// inputs. Names are a logical/schema concern; build and probe kernels consume +/// these stable positions. +struct JoinKeyColumns { + std::size_t left_index = 0; + std::size_t right_index = 0; + + auto operator==(const JoinKeyColumns&) const -> bool = default; +}; + +/// The complete column-name resolution for one join: positional key bindings +/// plus the authoritative output gather/rename plan. This is the join analogue +/// of `ColumnNameMap`: consumers resolve names once, then share this value +/// instead of independently looking them up or reconstructing output names. +struct JoinColumnMapping { + /// Ordered physical layouts against which every position below was + /// resolved. A lazy child can consume predicate-only columns before the + /// join boundary; execution detects that change once and re-resolves the + /// complete mapping against the concrete tables. + std::vector left_input_names; + std::vector right_input_names; + std::vector keys; + std::vector output; + + auto operator==(const JoinColumnMapping&) const -> bool = default; +}; + /// The single authority on a join's output column list and naming. /// /// IR schema inference, the materialized interpreter, the chunked executor and @@ -75,4 +102,14 @@ struct JoinOutputColumn { const JoinSuffixPolicy& suffix = {}) -> std::expected, std::string>; +/// Resolve every textual join key to its input position and compute the output +/// plan through `plan_join_output`. Known schemas call this during physical +/// planning; lazy/unknown schemas call it once when their concrete columns +/// first reach the build/probe barrier. +[[nodiscard]] auto resolve_join_columns(JoinKind kind, const std::vector& keys, + std::span left_names, + std::span right_names, + const JoinSuffixPolicy& suffix = {}) + -> std::expected; + } // namespace ibex::ir diff --git a/include/ibex/runtime/interpreter.hpp b/include/ibex/runtime/interpreter.hpp index 118686fe..5a86a6c4 100644 --- a/include/ibex/runtime/interpreter.hpp +++ b/include/ibex/runtime/interpreter.hpp @@ -703,6 +703,18 @@ struct ExecutionContext { /// chunked join, and the lazy-table keep-rows scan) and must agree. bool parallel_join_probe = true; + /// Child subtrees the materialized-call fallback already built through the + /// physical path (`build_materialized_fallback`), keyed by node pointer. + /// Null for every ordinary call. When set, `interpret_node` returns the + /// pre-built table for a node listed here instead of recursing into it, so a + /// filtered or projected input to a fallback breaker keeps the fused + /// parallel scan `build_operator` gave it rather than being re-evaluated + /// whole-table and serial. Only the fallback node's direct children appear; + /// deeper descendants are interpreted normally. Not owned — the tables live + /// in the fallback frame for the duration of the `interpret_node` call. + const std::vector>* pre_materialized_children = + nullptr; + /// Look up a deferred scan by its plan (instance) name, or null if there is /// no registry or no matching entry. [[nodiscard]] auto deferred_scan(const std::string& name) const -> const DeferredScan* { diff --git a/libs/data_gen/data_gen.ibex b/libs/data_gen/data_gen.ibex index a5d99201..ade43564 100644 --- a/libs/data_gen/data_gen.ibex +++ b/libs/data_gen/data_gen.ibex @@ -17,16 +17,24 @@ extern fn gen_ticks( volatility: Float64 = 0.5, interval_ms: Float64 = 1000.0, start_ts_ms: Int = 0 -) -> DataFrame from "data_gen.hpp"; +) -> DataFrame<{ + timestamp: Timestamp, + symbol: String, + price: Float64, + volume: Int64 +}> from "data_gen.hpp"; // Gaussian random walk: value[0] = start, value[i] = value[i-1] + N(0, step_std). -extern fn gen_walk(n: Int, start: Float64 = 0.0, step_std: Float64 = 1.0) -> DataFrame from "data_gen.hpp"; +extern fn gen_walk(n: Int, start: Float64 = 0.0, step_std: Float64 = 1.0) + -> DataFrame<{ value: Float64 }> from "data_gen.hpp"; // n iid samples from N(mean, stddev) in a single "value" column. -extern fn gen_normal(n: Int, mean: Float64 = 0.0, stddev: Float64 = 1.0) -> DataFrame from "data_gen.hpp"; +extern fn gen_normal(n: Int, mean: Float64 = 0.0, stddev: Float64 = 1.0) + -> DataFrame<{ value: Float64 }> from "data_gen.hpp"; // n iid samples from Uniform[low, high) in a single "value" column. -extern fn gen_uniform(n: Int, low: Float64 = 0.0, high: Float64 = 1.0) -> DataFrame from "data_gen.hpp"; +extern fn gen_uniform(n: Int, low: Float64 = 0.0, high: Float64 = 1.0) + -> DataFrame<{ value: Float64 }> from "data_gen.hpp"; // n sequential string ids "0" .. "(n-1)". extern fn gen_ids(n: Int, prefix: String = "row") -> DataFrame from "data_gen.hpp"; @@ -34,4 +42,5 @@ extern fn gen_ids(n: Int, prefix: String = "row") -> DataFrame from "data_gen.hp // Static reference data: one row per distinct symbol, with columns // symbol, name, sector, currency, lot_size, tick_size. Deterministic (no RNG), // so it joins against `gen_ticks` output on `symbol`. -extern fn gen_reference(symbols: String = "AAPL,MSFT,GOOG") -> DataFrame from "data_gen.hpp"; +extern fn gen_ids(n: Int, prefix: String = "row") + -> DataFrame<{ id: String }> from "data_gen.hpp"; diff --git a/plans/README.md b/plans/README.md index b860037b..810ef3d4 100644 --- a/plans/README.md +++ b/plans/README.md @@ -1,7 +1,8 @@ # Plans Index -Status of every plan in this directory, grouped by lifecycle. Statuses -re-verified against the source tree on **2026-08-27** (that pass also compacted +Status of every plan in this directory, grouped by lifecycle. Statuses were +initially re-verified against the source tree on **2026-08-27** (the +kernel-pipeline entry was re-verified again on 2026-08-29); that pass compacted the four largest plans — `parallelism-overview`, `query-shape-conformance`, `runtime-multithreading`, `kernel-pipeline-execution` — plus `owned-agg-per-chunk-barrier`, moving their measurement diaries to git history @@ -15,13 +16,13 @@ history). | Plan | Status | What's actually left | |---|---|---| | [beat-polars-plan.md](beat-polars-plan.md) | **Umbrella plan** for the multi-core push (proposed; §8 keeps the baseline record and dead ends). 2026-08-27 update corrected two stale q10 diagnoses (carried group fields optimized during discovery, ~−10.5%; the "36ms serial join build" was inclusive attribution). | Target: implied parallel fraction 44% → 60–65%. Workstreams W1 parallel inner join, W2 aggregate residue, W3 scheduler slice 2, W4 chunked `let` bindings, W5 small-query-tax guard. Points into pipelined-execution + runtime-multithreading + kernel-pipeline for mechanism. | -| [kernel-pipeline-execution-plan.md](kernel-pipeline-execution-plan.md) | **Phase 2 complete** except `KernelContext` (deliberately unbuilt); **Phase 3 started** (executor-owned ordered handoff, islands dissolved into a pipeline mode); **Phase 4 construction ownership done** (backlog 116→6 breakers, plan describes 97% of real-work nodes) but **decomposition not started** — operators are unchanged, so the exit criterion is not met. Architectural successor: typed logical IR, physical pipelines, morsel executor, templated kernel library; not a JIT. | Next: split the join operator into scheduled `HashBuild`+`HashProbe` (join build-side split DONE 2026-08-25, q21 −8.5%; probe-as-pipeline-step BUILT off-by-default, fires on 1/22 queries); aggregate phase decomposition (blocked-first on an already-broken determinism constraint — `try_owned_pair` vs the serial path disagree bit-for-bit at ≥65536 rows, no test); port `Tail`/`TopK`/`FilterHead`/`FilterTail`; split `chunked.cpp` by ownership. | +| [kernel-pipeline-execution-plan.md](kernel-pipeline-execution-plan.md) | **Phase 2 complete** except `KernelContext` (deliberately unbuilt); Phase 3 handoff/island/raw-thread work complete, with accounting and DOP/memory budgets deferred; Phase 4 construction ownership **and fan-out authority** done (backlog 116→6 breakers, plan describes 97% of real-work nodes). Streaming inner joins have typed `HashBuild`/`HashProbe` nodes and positional `JoinColumnMapping`. Streaming aggregates have positional `AggregateColumnMapping`, authoritative partition/finalize policy, and a typed Discovery → Accumulation → FinalOrdering → Emission hash-fallback chain. The serial coordinator invokes all four nodes through a bounded discovery transfer or explicit fused marker, with independent profile rows. Executor-seam mutations prove mappings, policies, and structural edges are consumed or rejected. Known closed schemas bind during planning; lazy/open schemas bind once at execution. Semi/anti retains its separate streaming operator. Architectural successor: typed logical IR, physical pipelines, morsel executor, templated kernel library; not a JIT. | Next: attach aggregate fan-out policy to each structural node, admit it phase by phase, then split `chunked.cpp` by ownership. | | [benchmark-perf-priorities.md](benchmark-perf-priorities.md) | Living reference | P0–P2 resolved/landed; rolling min/max optimized. Open: suite trimming (pin sqlite + data.table frollapply cells, duckdb at 3 scales); P4 `tanh` deferred pending accuracy-vs-speed call; P3 ohlc scatter-bound (negative result recorded — don't re-attempt naive fusion); re-check rolling_mean on AWS after the July 2026 regression fix | | [benchmark-coverage-plan.md](benchmark-coverage-plan.md) | ~95% done | #9 ClickHouse EWMA (needs arrayFold workaround); #10 DataFusion `fill_forward/backward` + `tf_asof_join` | | [count-window-plan.md](count-window-plan.md) | Implemented (interpreter + codegen) | Per-call count/duration windows work (`__window_n`/`__window_ns` in lower.cpp + window.cpp), and the compiled path (`ibex_compile`) is at parity. Open: `window N rows` block syntax and tuple-field `update` inside `window` (interpreter doesn't support that combo either, so codegen correctly still rejects it). The old monotonic-deque follow-up for `rolling_min`/`rolling_max` is done. | | [non-row-local-filter-plan.md](non-row-local-filter-plan.md) | Stage 1 shipped | `lag`/`lead`/`is_null` in filter work. Remaining: `rank(...)` in filter/select with `by`, explicit `order {}` context, rolling functions in filter (`price > rolling_mean(price)`) | | [bigger-than-ram-plan.md](bigger-than-ram-plan.md) | Phase 4 bullet 1 of 4 done | Out-of-core execution. Done: chunked/streaming `read_parquet` (branch `chunked-parquet-read`; ~6.5× lower peak RSS, ~1.7× faster, verified local + AWS). Next: column projection pushdown, row-group stats pushdown, directory/Hive datasets (rest of Phase 4), then Phase 1 spill infrastructure (prerequisite for Phases 2–3, 6–7: external sort, out-of-core join, adaptive spill selection) | -| [runtime-multithreading-plan.md](runtime-multithreading-plan.md) | Phase 1 landed and **ON by default** (parallel islands = two-phase filter + metadata-tail, every shape beats serial); Phase 3a complete; Phase 3b first source slice landed; Phase 4 items 1–2 landed, item 3 RETIRED (the join gap was Categorical probe keys hashed as *text*, not threading), item 4 part-done. **Nomenclature: `IBEX_THREADS` → `IBEX_CORES`; `IBEX_PARALLEL` removed (serial is `IBEX_CORES=1`).** | The PDS-H multithreading gap is **Phase 4 (parallel barriers)**, not sources — only 5/22 queries even form an island. Next: group-by string/int/generic hash paths + `distinct`; the LazyTable Synchronization Contract (written, unimplemented — Phase 3b's foundation); Phase 2 deterministic RNG (designed, not started). Re-measure at a larger scale before ranking — the threading share of a gap grows with row count. | +| [runtime-multithreading-plan.md](runtime-multithreading-plan.md) | Row-local morsel-parallel pipelines are **ON by default**; Phase 3a is complete; Phase 3b's first source slice landed; Phase 4 items 1–2 landed, item 3 RETIRED (the join gap was Categorical probe keys hashed as *text*, not threading), item 4 part-done. **Nomenclature: `IBEX_THREADS` → `IBEX_CORES`; `IBEX_PARALLEL` removed (serial is `IBEX_CORES=1`).** | The PDS-H multithreading gap is **parallel barriers**, not sources. Next: group-by string/int/generic hash paths + `distinct`; the LazyTable Synchronization Contract (written, unimplemented — Phase 3b's foundation); Phase 2 deterministic RNG (designed, not started). Re-measure at a larger scale before ranking — the threading share of a gap grows with row count. | | [join-perf-plan.md](join-perf-plan.md) | Items 1–3 done (2026-07-14; q09 −23%, q13 −30%) | Join/group-by performance findings; see the file's Results section. beat-polars points here for join mechanism | | [owned-agg-per-chunk-barrier-plan.md](owned-agg-per-chunk-barrier-plan.md) | High-cardinality partition-owned aggregation vs Polars streaming. q18 (single-Int64 `Sum`) largely closed by the async hot/cold rewrite (**−33%**); the parallel finalize merge landed for all owned paths (**−7%**); q21's ordered-run `Count` finalize + emit fusion landed (**−11.2% SF-4**). | q20/`PairIntKey` (the hot table doesn't help a scattered composite key — a per-partition `CardinalitySketch` is the candidate); q21's remaining wall is the per-chunk accumulate orchestration + a duplicate lineitem decode + a 40ms serial hash-join build. **Do NOT touch `part_count` or serial-`reserve` the maps** — both measured dead ends. | | [grouped-chunkview-update-plan.md](grouped-chunkview-update-plan.md) | Mostly complete — `update …, by k` runs off an immutable `GroupedRowPlan` (CSR) instead of gather → per-group `Table` → scatter. Sub-plan of kernel-pipeline Phase 2. | Remaining materialized shapes: `rank`, variable-width ordered state, `window`-clause `lag`/`lead`. | @@ -40,7 +41,6 @@ history). | Plan | Notes | |---|---| -| [pipelined-execution-plan.md](pipelined-execution-plan.md) | **Phase 2 partially implemented.** Multi-chunk correctness, lazy row-group streaming, concurrent scan decoding, ordered source→map overlap, and a bounded streamed join-probe handoff landed. SF-1 improves 2.5–4.6% at 2–8 cores; SF-4 improves 0.6–3.7% at 4–8. The two-core SF-4 crossover (+6.9%) and q18 are explicit follow-ups. Open: progress-aware admission/backpressure and general breaker scheduling | | [parallel-chunkview-output-plan.md](parallel-chunkview-output-plan.md) | ChunkView kernel output protocol under parallel execution — `update_table` stays the semantic/ownership authority, a direct kernel plans a field and writes one assigned range. Sub-plan of kernel-pipeline Phase 2 (see the "Parallel chunk updates" entries). | | [radix-partitioned-groupby.md](radix-partitioned-groupby.md) | Noted, not built. High-cardinality group-by is memory-bound; radix partitioning remains a q18/q20 mechanism. Q10 no longer reaches the generic mixed-key ceiling (2026-08-27: FD reduction + discovery-time `First` gathering handle that shape). | | [exists-subquery-plan.md](exists-subquery-plan.md) | Proposal: `exists(table_expr)` as a boolean subquery term — semi/anti/mark joins and the residual-predicate case | diff --git a/plans/beat-polars-plan.md b/plans/beat-polars-plan.md index d8e76e14..ada80017 100644 --- a/plans/beat-polars-plan.md +++ b/plans/beat-polars-plan.md @@ -4,11 +4,11 @@ Status: **proposed.** Written 2026-08-16 on branch `streaming-scan-units`, after the pipelined-execution Phase 2 slices (streamed scan units, concurrent decode, source→map pipeline, join-probe handoff) landed. This is the umbrella plan: it sets the target, decomposes the remaining gap into workstreams with expected -payoffs, and sequences them. It supersedes nothing — it points into -`pipelined-execution-plan.md`, `runtime-multithreading-plan.md`, and -`join-perf-plan.md` for mechanism (the former fourth input, -`chunked-execution-plan.md`, was removed from the tree 2026-08-22; its -open items live in the kernel-pipeline plan and bigger-than-ram). +payoffs, and sequences them. It points into +`kernel-pipeline-execution-plan.md`, `runtime-multithreading-plan.md`, and +`join-perf-plan.md` for mechanism. The former pipelined- and chunked-execution +plans were removed as superseded; their remaining work lives in the kernel +pipeline plan and bigger-than-ram plan. Absorbed `pds.md` (the 2026-08-11 status snapshot this grew out of) on 2026-08-22: §5 and §6 took its still-unique traps and dead ends, §8 keeps its baseline record. @@ -193,8 +193,8 @@ is, so every workstream is stated as "serial ms attacked". ## 2. Where the serial time is Calling-thread self-ms summed over the suite at 8 cores, with worker help drawn -(`pool_work / self`), from `pipelined-execution-plan.md` ("Where the serial -time actually is", post-semi-join numbers): +(`pool_work / self`), from the archived pipelined-execution measurement study +(git history; post-semi-join numbers): | operator | self ms | pool ms | worker help | verdict | |---|---:|---:|---:|---| @@ -512,7 +512,7 @@ W3.1 progress-aware admission ── next scheduler slice; fixes 2-core cl W3.2 q18/q22 rundown ── decides W4's priority W4 chunked let bindings ── design note first; largest structural risk W3.3+ aggregate-output stages, - general scheduler, island retirement + explicit breaker phases and general scheduler W2.3 parallel gid ── only if still dominant after the above ``` diff --git a/plans/bigger-than-ram-plan.md b/plans/bigger-than-ram-plan.md index ce5c55f6..4ff92f86 100644 --- a/plans/bigger-than-ram-plan.md +++ b/plans/bigger-than-ram-plan.md @@ -72,8 +72,8 @@ built on it. Flag this as the one open design question worth confirming before writing code: bespoke binary would be marginally faster to (de)serialize but is a new format to maintain and test independently. -- A byte-budget tracker (`IBEX_MAX_MEMORY`, mirroring the `IBEX_THREADS` knob - design in `runtime-multithreading-plan.md`): unset/0 means unbounded — the +- A byte-budget tracker (`IBEX_MAX_MEMORY`, following the current compute/pool + configuration split in `runtime-multithreading-plan.md`): unset/0 means unbounded — the default, so existing benchmarks and RSS baselines in `plans/benchmark-perf-priorities.md` are completely unaffected until a user opts in. When set, operators accumulate in memory and spill only once the diff --git a/plans/kernel-pipeline-execution-plan.md b/plans/kernel-pipeline-execution-plan.md index 3d820c50..d03569dd 100644 --- a/plans/kernel-pipeline-execution-plan.md +++ b/plans/kernel-pipeline-execution-plan.md @@ -2,10 +2,14 @@ **Status: in migration.** Phase 0 resolved by disposition; Phase 1 landed 2026-08-22; Phase 2 complete except `KernelContext` (deliberately unbuilt); -Phase 3 started; Phase 4 construction-ownership done, decomposition not started; -Phase 5 not started. **Compacted 2026-08-27** — the ~40-entry Phase 2 per-commit -diary is in git history at the pre-compaction commit's parent; the "Where Phase -2 stands" table below is the current state. +Phase 3's handoff/island/raw-thread work is complete, with accounting and +DOP/memory budgets deferred; Phase 4 construction ownership and parallelism +authority are done, and the targeted join and aggregate decomposition is +complete; Phase 5 is in progress: fused logical node kinds are retired and the +aggregate and streaming inner-join execution families are now outside the monolith. **Compacted +2026-08-27** — the ~40-entry Phase 2 per-commit diary is in git history at the +pre-compaction commit's parent; the "Where Phase 2 stands" table below is the +current state. The goal is **feature parity on this architecture**: every shape the old execution seams supported reaches either a kernel or an explicit, inspectable @@ -22,9 +26,13 @@ canonicalize table is in `include/ibex/ir/canonicalize.hpp`. ## Why -`src/runtime/chunked.cpp` (~13k lines) is the physical planner, most streaming -operator implementations, parallel islands, pipelined stages, and a large set of -operator-specific eligibility rules — all grown together because +`src/runtime/chunked.cpp` remains the residual streaming operator +implementations and a large set of operator-specific construction rules. +Planning lives in `physical_plan.cpp`, migrated-plan validation and dispatch in +`physical_executor.cpp`, and generic map/morsel execution in +`pipeline_executor.cpp`. Aggregate and streaming inner join have moved to +family-owned translation units, but the remaining responsibilities are still +grown together because `build_operator(const ir::Node&)` lowers logical nodes directly into mutable `Operator::next()` objects. Three costs: (1) a physical choice has no representation ("stream this join", "materialize this aggregate" are builder @@ -102,22 +110,26 @@ parser AST → typed logical IR → physical plan → pipeline executable and profiled — preserves median/quantile/EWMA/predicates/reshape until a physical implementation exists. -## Where this stands (2026-08-24 / -25) +## Where this stands (re-verified 2026-08-29) **Done:** Phase 1 (physical plan exists, inspectable); Phase 2 (map kernels ported, fusion is physical, fused node kinds retired as an execution concern); Phase 3 items 1/3/4 (one executor-owned ordered handoff, islands dissolved into a pipeline mode, no raw-thread branch concurrency); Phase 4 **construction -ownership** (every breaker PDS-H reaches is built from the plan — backlog -116→6 breakers, plan describes 97% of real-work nodes). - -**The 97% flatters it:** that measures who *constructs* operators, not how they -are shaped. The operators are unchanged — a join is still one -`ChunkedInnerJoinOperator`, not `HashBuild` feeding a separate `HashProbe` -across a barrier; the aggregate is still one operator, not four phases. So a -probe can't be a pipeline step (can't fuse with filters/projections above it), -one build can't feed several probes, and the aggregate's phases can't be -scheduled or measured separately. +ownership, parallelism authority, and targeted decomposition** (every breaker +PDS-H reaches is built from the plan — backlog 116→6 breakers, plan describes +97% of real-work nodes; Distinct, streaming Join, and streaming Aggregate read +resolved fan-out policy from the plan; join and aggregate expose explicit +execution phases). + +**What the 97% contains:** the streaming inner join is shaped as explicit +`HashBuild` and `HashProbe` nodes across a typed runtime-oriented barrier. The +hash-aggregate fallback now has typed discovery, accumulation, final-ordering, +and emission plan nodes. A serial coordinator invokes all four: discovery +publishes a bounded per-chunk transfer that accumulation consumes before the +source advances, while owned/async one-pass kernels publish an explicit fused +result. Each node has its own execution-profile row. Semi/anti retains its +separate streaming operator. ### Next, in order @@ -132,14 +144,17 @@ scheduled or measured separately. plan-execution time. Measured: q21 −8.5% from the parallel fill; scheduled invocation geomean −1.9% over 22 queries (q21 +5.1%, q19 +5.5% reported not averaged). Kill switches `IBEX_JOIN_BUILD_SERIAL` / `IBEX_JOIN_BUILD_LAZY`. -2. **Make the probe a step inside a map pipeline — BUILT 2026-08-25, off by - default** (`f8e84446`, `7cc940b1`). Correct at 1/2/8 cores. **Fires on 1 of - 22 queries** (q17), so it can't be measured — no performance claim. +2. **Make the probe a step inside a map pipeline — DONE structurally.** The + eligible probe is now admitted into `build_morsel_worker_chain` and installed + before the row-local map steps; `build_pipeline_from_input` extracts the + probe side rather than materializing join output first. This preserves the + intended source → probe → map worker shape, including swapped-mode coverage. + `HashProbe` is now also an explicit physical-plan and executor node; this + still carries no end-to-end performance claim. Preconditions met: build is a scheduled phase (`8cb4e936`), build side is jointly owned so N probes share one (`6df9a966`), probe is its own operator - (`177b7a93`, `JoinProbeOperator`). What remains: admit it into - `build_morsel_worker_chain`, which today takes only row-local `MapStep`s — a - probe is 1:N and the chain builder has no word for that. + (`177b7a93`, `JoinProbeOperator`). The physical promotion is complete; the + opt-in map-chain fusion remains the measured construction described here. - **Coverage is the finding.** PDS-H join modes: 28 `Stream`, 12 `Precomputed`, 11 `Swapped`. `Precomputed`/`Swapped` materialize both sides and emit one table — no probe pipeline to give. `Swapped` (a third of @@ -153,17 +168,45 @@ scheduled or measured separately. NOT dominate (14% of join self-time, ≤10.5% of any query's wall) — retiring the standing "assemble_output dominates" note. So the fusion argument is **structural, not performance**. - - **Test gap:** the extraction shipped a deterministic q18 segfault that all - 1756 ctests passed through — the suite has no deferred-probe join whose - resolved right falls under `kStreamRightThreshold`. `check_answers.py` - caught it 21/22. Close before the morsel-chain work. + - **Deferred-probe threshold regression — DONE 2026-08-29.** The focused + test starts with a 70k-row deferred right, publishes a build-side + membership filter that resolves it to one row, then exercises the + below-`kStreamRightThreshold` BuildRight path after the 20k-row left was + drained. It asserts filter publication, parallel probe activation, and + serial/parallel structural equality. 3. **Phase 4 aggregate decomposition** — discovery / per-partition slots / final ordering / emission as phases. **Determinism blocker cleared 2026-08-27** - (see below — the divergence no longer reproduces; guard test landed). Slice 1 - (observability: `partition` + `finalize` phases on the plan, `explain - physical` prints them, `check_agg_plan` aborts on disagreement) LANDED. - Slices 2–3 move the authority (delete the operator's open-coded floors + - `min(budget, pool, 64)` caps). + (guard test landed). The preflight is complete: all four structural-node + policies appear in `explain physical`; aggregate fan-out gates read their + node's resolved policy; `AggregateColumnMapping` binds known schemas + during planning and lazy/open schemas once at execution; and physical-plan + mutation tests prove mapped positions, typed edges, and worker ceilings are + consumed or rejected. **Structural plan slice DONE 2026-08-29:** the hash + fallback carries typed Discovery → Accumulation → FinalOrdering → Emission + nodes and ownership edges; `explain physical` renders the chain and the + executor rejects missing, redirected, or mistyped edges. **Serial lifecycle + slice DONE 2026-08-29:** final ordering is no longer triggered implicitly by + emission; a coordinator drains the input, invokes the deterministic ordering + merge, then permits output construction. Discovery and accumulation remain + deliberately fused only where owned/async or specialized categorical kernels + produce final aggregate state in one pass. + **Discovery-transfer/accounting slice DONE 2026-08-29:** ordinary paths pass + group IDs, positional aggregate entries, row count, and any seeded-First mask + through one chunk-bounded `AggregateDiscoveryTransfer`; fused kernels return + an explicit fused marker. The coordinator invokes Discovery and Accumulation + separately before advancing the source, and all four structural nodes have + independent profile rows. Release A/B over `groupagg,multi,events` (7 + interleaved repeats, 3 timed iterations, pinned core) found all nine query + deltas noise; final total +0.22%, geometric speedup 0.992×. + **Structural fan-out authority DONE 2026-08-29:** the coarse `partition` / + `finalize` records are removed. Discovery, Accumulation, FinalOrdering, and + Emission each carry and supply their own policy; explain renders all four, + and profile-backed mutations prove the executor consumes their individual + worker ceilings. Data-derived morsel/partition counts and specialization + thresholds remain next to the kernels that can observe them. Release A/B + against the preceding commit over `groupagg,multi,events` (7 interleaved + repeats, 3 timed iterations, pinned core) classified all nine deltas as + noise; total +0.53%, geometric speedup 0.998×. 4. **Port `Tail` / `TopK` / `FilterHead` / `FilterTail` — DONE.** Same single-operator shape as Order/Head: `plan_physical` marks each migrated, `build_physical_{tail,topk,filter_head_tail}` construct them (moved verbatim @@ -171,13 +214,29 @@ scheduled or measured separately. physical` renders `Breaker() serial (single-operator breaker, no fan-out point)`. TopK stays a serial bounded-heap select by design. No behaviour change. -5. **Phase 5 item 1 — split `chunked.cpp` by ownership** — easier the more of - Phase 4 has landed. -6. **Sweep process-global plan counters in tests** — one test passed while its - premise was false (`physical_materialized_calls` is process-wide, other tests - in the binary bump it). Others may lean the same way. +5. **Phase 5 item 1 — split `chunked.cpp` by ownership — IN PROGRESS.** Aggregate, + streaming inner join, physical-plan dispatch, and the generic pipeline/morsel + executor are extracted. The pipeline unit owns worker chains, ordered handoff, + two-phase filter, deferred-scan pipeline, asynchronous stage, and source + strategy; concrete row-local factories remain callbacks owned by their + operator families. No extra call was added to `Operator::next()`. Semi/anti, + materializing joins, and residual breaker families remain in their existing + owners. **Phase 5 item 3 DONE 2026-08-29** (`5f7afc59`, `94957719`, + `d1204b63`; `plans/physical-fallback-adapter-plan.md`): the 15-branch + materializing per-kind switch in `build_operator_impl` is gone. Every + non-migrated kind resolves through one `build_materialized_fallback` → + `interpret_node`; `build_materialized_fallback` still builds the breaker's + relational inputs through `build_operator` (handed back via + `ExecutionContext::pre_materialized_children`) so a filtered/projected input + keeps its fused parallel scan. `explain physical` names the retained subtree. + `chunked.cpp` −290 lines net. Next: physical-plan-migrate individual fallback + kinds as `physical_fallbacks_for(kind)` ranks them. +6. **Sweep process-global plan counters in tests — DONE 2026-08-29.** The + formerly false-premise test now checks the migrated pipeline counter. All + three remaining counter assertions take a local before/after delta, and no + test reads `physical_materialized_calls`; no counter redesign is required. 7. **Phase 3 item 5 — per-pipeline scheduling accounting** — small, worth more - after 1–3. + once the join and aggregate phases have independent identities to attribute. 8. **Phase 3 item 2 — DOP/memory budgets** — analysed and **blocked** (`phase3-dop-budget-analysis.md`): the pool is 65% idle with nothing queued, so a budget rations a non-scarce resource. Reopen when a multi-producer @@ -342,8 +401,9 @@ breaker operators (Phase 4). The **stronger** failure model is now shared (sequence-ordered, allocation- free `record_fault`, worker liveness through an exit guard — the scan pipeline's first-writer-wins exception path is gone). Naming followed: - `ParallelIslandOperator` → `MorselPipelineOperator`, stats keys unchanged - (tooling reads them). **Zero "island" occurrences in `src/`/`include/`/`tests/`.** + The prior island executor was replaced by `MorselPipelineOperator`; stats + keys stayed unchanged because tooling reads them. **Zero "island" occurrences + in `src/`/`include/`/`tests/`.** `PipelinedStageOperator` keeps its raw thread + plain `std::deque` FIFO (cap 2, single producer) — deliberately not merged (no sequence ordering to maintain). @@ -353,9 +413,10 @@ breaker operators (Phase 4). 4. **Eliminate raw-thread construction from join/builder branches — CLOSED, by deletion.** Both sites (build overlapped with materialize on a raw thread) measured worse (q09 +57%, then +47.5% under a since-removed helper-thread - budget; q10 ~−3% didn't survive widening) and were reverted. `chunked.cpp:10002` - / `:13066` carry the measurements. Branch concurrency needs a **cost-aware** - gate, not a thread-count one. The only remaining non-pool thread is + budget; q10 ~−3% didn't survive widening) and were reverted. The named join + builder branches retain the rationale; git carries their former locations. + Branch concurrency needs a **cost-aware** gate, not a thread-count one. The + only remaining non-pool thread is `PipelinedStageOperator`'s (item 1's subject). 5. **Per-pipeline scheduling accounting** — not started, worth more after the join/aggregate splits give it phases to attribute to. @@ -363,11 +424,11 @@ breaker operators (Phase 4). **Concurrency-ownership inventory:** raw threads — `WorkerPool` (sanctioned) + `PipelinedStageOperator` (long-lived, blocks on ring backpressure, `StageThreadScope` for the profiler). Bounded handoffs — the two sequence rings -(now `OrderedChunkRing`) + the stage FIFO + the pool's own. 41 `pool.submit` -sites (`chunked.cpp` 25) — DOP is seized there; `WorkerPool::submit` calls -`invariant_violation` from a pool thread and 29 sites check -`on_worker_pool_thread()` first (what makes nested parallelism a crash not a -deadlock). +(now `OrderedChunkRing`) + the stage FIFO + the pool's own. Pool submissions +remain concentrated in the pipeline executor and breaker families — DOP is +seized there; `WorkerPool::submit` calls `invariant_violation` from a pool thread +and callers guard nested submission with `on_worker_pool_thread()` (what makes +nested parallelism a crash rather than a deadlock). ### Phase 4 — migrate the high-value breakers @@ -378,21 +439,23 @@ remaining are materializing joins (`nulls equal` / `expect` / non-equi predicates — porting them ports the semantics, not the construction). **Construction ownership DONE** (Join streaming `f5610646`, Aggregate `902d6941`, -Order `ececc75f`, Head/Distinct `49ca33c1`). **Decomposition NOT started** — the -operators are unchanged; the branches moved into `build_physical_join` / -`build_physical_aggregate` rather than dissolving into pipeline stages, so the -exit criterion ("fast paths no longer depend on special builder branches") is -**not met**. A breaker's parallelism (fan-out decision, worker cap, row floor, -partition strategy) is still private to `chunked.cpp`, invisible to `explain -physical`. +Order `ececc75f`, Head/Distinct `49ca33c1`). **Parallelism authority is also +DONE**: Distinct, streaming Join, and streaming Aggregate receive resolved +`BreakerParallelism` from the plan rather than deriving their worker caps and +fan-out permission privately. **Targeted decomposition is DONE**: streaming +inner join is an explicit HashBuild → HashProbe execution shape, and aggregate +is an explicit Discovery → Accumulation → FinalOrdering → Emission lifecycle. +Both execution families now live outside `chunked.cpp`; the remaining Phase 5 +work is separating the generic physical-plan adapter and morsel executor from +the residual operator families. **The decomposition target is specified in [`src/runtime/PARALLELISM.md`](../src/runtime/PARALLELISM.md), "Target: parallelism as a plan decision"** — the `BreakerParallelism` descriptor, the planner-vs-operator split (same one `JoinPlan` already made), the `explain physical` format, the observability-before-authority slicing, and the sequence -(Distinct → Order/TopK → Join → Aggregate, the last blocked on the determinism -reconciliation). +(Distinct → Order/TopK → Join → Aggregate; the determinism reconciliation is +complete, so the remaining blocker is structural decomposition). *Method note (decided the outcome twice):* each port = name the builder's own predicates + de-duplicate, have the planner **relay** them, have the seam @@ -405,29 +468,135 @@ at all (a one-valued strategy enum would be ceremony). 1. **Hash join** — construction DONE; **data side DONE** (`5918b5cc`, `8a644381`, `f6a1a632` — build returns an immutable `JoinHashIndex`; `JoinProbe` consumes one via `shared_ptr` so writing build state during a probe is a - compile error); **operator side NEXT** (two scheduled operators — see "Next" - item 2). NOT blocked on a cost model (corrected 2026-08-25). -2. **Hash aggregate** — construction DONE; phase decomposition NOT started, - blocked-first on the determinism divergence above. + compile error); **map-pipeline probe fusion DONE**; **explicit physical + `HashBuild`/`HashProbe` DONE 2026-08-29**. The build produces a move-only + `HashProbeInput` whose variant fixes Stream / Swapped / Precomputed + orientation; the physical probe consumes it, and the temporary coordinator + is discarded at the barrier. Semi/anti deliberately retains its separate + streaming operator. **Column binding follow-up DONE 2026-08-29:** + `JoinColumnMapping` resolves mapped left/right keys to positions together + with the authoritative output plan; known closed schemas bind in the + physical planner, while lazy or layout-dynamic inputs bind once at the + concrete barrier; probe kernels no longer look columns up by textual key per + chunk. The mapping records both input layouts and re-resolves the complete + key/output mapping when predicate pushdown or projection pruning narrows a + concrete child. Matching-layout mutations are rejected rather than hidden by + rebinding. This fixes the q14/q19 crash where valid key positions masked a + stale `JoinOutputColumn::source_index`. **Extraction DONE 2026-08-29:** hash + build state, orientation, probe kernels/operators, gather assembly, and + deferred-probe resolution live in `join_chunked.cpp`. `chunked.cpp` retains + the generic morsel adapter through `JoinProbeFactory`; semi/anti and + materializing fallbacks did not move. NOT blocked on a cost model. +2. **Hash aggregate** — construction, positional column binding, fan-out + authority, physical-plan mutation coverage, and the four-node structural + hash-fallback chain DONE. Serial orchestration, the bounded + discovery→accumulation transfer, fused-result marker, final-ordering/emission + split, and per-node accounting are also DONE. The former determinism blocker + is resolved. `StreamingSorted` is the historical name for an adaptive + strategy: sorted group-at-a-time when possible, hash fallback otherwise + (including ordinary generated tables). Each structural node now owns its + fan-out policy. **Extraction DONE 2026-08-29:** the whole adaptive + sorted/hash family now lives in `aggregate_chunked.cpp` behind one private + factory. No hot loop or state boundary was split across translation units. + `AggregateColumnMapping` also records the layout it was resolved against: a + lazy child that consumes a predicate-only column may rebind once at its + concrete boundary, then every phase remains positional. This fixes the q01 + regression introduced when a logical closed schema was mistaken for a fixed + physical layout. 3. **Distinct + ordered** — construction DONE; `Tail`/`TopK`/`FilterHead`/ `FilterTail` ported too (see "Next" item 4). The whole Head/Tail/TopK/Filter* family and Distinct/Order now leave the per-kind switch. 4. Delete the `chunked.cpp` classes only after the physical path handles every - supported shape and the fallback is mutation-tested. Blocked on 1–2. + supported shape and the fallback is mutation-tested. The aggregate classes + and streaming inner-join classes are now deleted from `chunked.cpp`. ### Phase 5 — retire the monolith, simplify IR -1. Split by ownership: `physical_planner`, `pipeline_executor`, `kernels/`, one - file/family per breaker. +1. Split by ownership: `physical_plan`, `physical_executor`, + `pipeline_executor`, `kernels/`, one file/family per breaker. **IN + PROGRESS:** aggregate, streaming inner join, physical-plan execution, and the + generic pipeline/morsel executor are complete; planning lives in + `physical_plan.cpp`. Residual breaker-family extraction remains. 2. Move logical fusion/selection out of `ir::NodeKind` — **DONE** for `FilterProject` / `FilterUpdateProject`: both legacy types and their compatibility lowering are deleted. 3. Remove obsolete `build_operator` recursion; migrate `interpret_node` to an - explicit physical fallback adapter. + explicit physical fallback adapter. **DONE 2026-08-29** (`5f7afc59`, + `94957719`, `d1204b63`; `plans/physical-fallback-adapter-plan.md`). The + materializing per-kind switch is one `build_materialized_fallback` seam; + `build_operator` no longer recurses through a fallback subtree (only through + the pre-built relational inputs). `interpret_node` keeps its own recursion as + the fallback interpreter. 4. Make planner / executor / kernel tests independently runnable. Exit: `chunked.cpp` no longer exists as a monolithic execution/planning unit. +### Follow-up sequence + +1. **Deferred-probe threshold regression — DONE 2026-08-29.** Force a deferred + right side to resolve below `kStreamRightThreshold`, then assert serial and + parallel byte-identity and that the deferred build/probe path is reached. +2. **Explicit physical `HashBuild` and `HashProbe` nodes — DONE 2026-08-29.** + The data-only plan has distinct typed nodes connected by a + `RuntimeOrientedBuildOutput` edge; both retain the candidate inputs, and + `build_physical_join` consumes their policies. At execution the build moves + a Stream / Swapped / Precomputed `HashProbeInput` across that edge and the + probe owns all downstream work; the enclosing coordinator is discarded. + Edge mutations are rejected by the same validator execution calls, and + materializing plus semi/anti shapes carry no inner-join edge. The immediate + name-resolution audit is also complete: the edge carries one + `JoinColumnMapping` (positional keys + output provenance), resolved at plan + time when possible and once at execution otherwise. +3. Split aggregate execution at its existing ownership boundaries — discovery / + partition accumulation / final ordering / emission — first with serial + orchestration and plan-shape/accounting tests, then admit fan-out one phase + at a time with byte-identity checks. The typed plan shape and edge-mutation + tests, serial orchestration, bounded discovery transfer, fused marker, and + independent profile accounting are complete. Each structural node now owns + and supplies its fan-out policy, with byte-identity and profile-backed + worker-ceiling mutations. This step, including extraction of the resulting + aggregate family, is complete. +4. Add per-phase scheduling accounting only after steps 2–3 provide stable + pipeline identities. Keep DOP/memory budgeting blocked unless those changes + produce measured queue contention or a multi-producer consumer. +5. Move the resulting planner, executor, kernels, join, and aggregate families + out of `chunked.cpp`; replace residual recursion with the explicit physical + fallback adapter, preserving mutation-tested `MaterializedCall` coverage. + **Aggregate DONE 2026-08-29.** Correctness: focused physical tests plus all + 1,815 non-slow tests pass; SF4 q01 is byte-identical at one and eight cores. + Performance: the generated `groupagg,multi,events` A/B classified all nine + deltas as noise (total −2.33%); a fixed-but-unextracted SF4 baseline versus + the extracted target classified q01/q13/q22 all the same (geomean +0.3%, + byte-identical). Widened q01 alone was also a wash (+1.4%, p=0.478). + **Streaming inner join DONE 2026-08-29.** Correctness: all 1,815 non-slow + tests, focused physical/join/deferred-probe tests, and SF4 q05/q09/q14/q19/q21 + pass. The strict GCC runtime build passes. Performance: all four generated + join cases classified as noise (total −0.34%); ten join-heavy SF4 queries + were byte-identical and a wash (geomean −0.3%, every query under the 2% + practical floor). The opt-in `IBEX_PROBE_MORSELS=1` POC retains a + pre-existing SF4 q09 stall in both baseline and extracted trees; it remains + disabled and is a separate correctness follow-up. + **Physical-plan executor DONE 2026-08-29.** `physical_executor.cpp` owns root + validation, migrated-kind dispatch, path accounting, and + `build_operator_from_physical_plan`; concrete operator factories stay with + their implementations. Correctness: focused physical tests (29 cases, 4.34M + assertions), all 1,815 non-slow tests, and the strict GCC runtime build pass. + Performance: the broad + generated A/B total was −2.00%; a replica-controlled 18-query + core/groupagg/join run classified every delta as noise (total +0.77%), and a + 31-repeat join confirmation classified all four joins as noise (total + +0.15%). + **Pipeline/morsel executor DONE 2026-08-29.** Worker-private map chains, the + bounded ordered ring, two-phase filter, deferred-scan pipeline, asynchronous + stage, and source-strategy orchestration now live in + `pipeline_executor.cpp`; `chunked.cpp` supplies concrete map factories and + residual breaker construction through a narrow internal interface. + Correctness: focused pipeline/physical tests (24 cases, 4.34M assertions), + all 1,815 non-slow tests, the strict GCC runtime build, and debug/Release + Parquet + LightGBM plugin builds pass. Performance: a replica-controlled + `pipeline,filter,join` A/B (15 interleaved repeats) classified all 13 deltas + as noise; total +0.24%. + ## Acceptance gates (every phase, before the next starts) Full parser/IR suite; byte-identical serial + parallel across chunk grains diff --git a/plans/physical-fallback-adapter-plan.md b/plans/physical-fallback-adapter-plan.md new file mode 100644 index 00000000..e342491a --- /dev/null +++ b/plans/physical-fallback-adapter-plan.md @@ -0,0 +1,141 @@ +# Explicit physical fallback adapter + +Phase 5 item 3 of [`kernel-pipeline-execution-plan.md`](kernel-pipeline-execution-plan.md): +*"Remove obsolete `build_operator` recursion; migrate `interpret_node` to an +explicit physical fallback adapter."* Also tracked there as "Next" item 5 and +follow-up-sequence item 5 ("replace residual recursion with the explicit physical +fallback adapter, preserving mutation-tested `MaterializedCall` coverage"). + +## Problem + +`build_operator_impl` (`src/runtime/chunked.cpp` ~4923-5405) still carries a +~430-line per-`NodeKind` `if`-chain below the physical-plan seam. The seam itself +is already clean: + +```cpp +const physical::Plan plan = physical::plan_physical(node, registry, externs); +if (plan.migrated) return build_migrated_physical_operator(...); // join, agg, order, + // distinct, head/tail/topk, + // map-pipeline +physical::note_materialized_call(plan.reason, node.kind()); +// ... ~430 lines of per-kind if-chain ... +// tail: +auto table = interpret_node(node, registry, scalars, externs, exec, model_out); +return make_table_source(std::move(table.value())); +``` + +The `if`-chain splits three ways: + +| Bucket | Kinds | Branch behaviour | +|---|---|---| +| **A. Materializing breakers** | `Columns, Melt, Dcast, Cov, Corr, Transpose, Matmul, Resample, Window, AsTimeframe, Model, Construct, Stream, Program`, materializing `Join`, non-row-local `Update` | recurse `build_operator(child)` → `materialize_operator` → table fn → `make_table_source` | +| **B. Genuinely-streaming sources** | `Scan`+`stream_scans` (deferred lazy scan), `ExternCall` | build a real chunked / pipelined source operator | +| **C. Near-dead map duplicates** | `Filter, Project, Rename` | build row-local map operators, recursing into `build_operator` | + +Confirmed: `interpret_node` has a branch for every bucket-A kind and recurses into +itself for children, so bucket A hand-rolls child recursion that `interpret_node` +already does. The function tail *is* the adapter — bucket A only needs to reach +it. + +`explain physical` prints `MaterializedCall()` with no node identity, +contradicting the header contract (`physical_plan.hpp:36`: *"`MaterializedCall` +naming the logical subtree retained by the fallback"*). + +## Steps + +### Step 1 — Name the `MaterializedCall` node (observability first) — DONE `5f7afc59` + +`explain_physical` (`physical_plan.cpp` ~760) emits the root `NodeKind` in the +`MaterializedCall(...)` line, e.g. `MaterializedCall(Melt: root is not a +row-local map)`. `plan.root` is always set (`plan_physical` line 509). Reuse +`node_kind_name_impl`. Update the three affected assertions in +`tests/test_physical_plan.cpp` (lines ~562, ~760, and the join-materializing +block ~1138). No execution change. Landed first, on its own commit. + +### Step 2 — Collapse bucket A into the tail adapter — DONE `94957719` (pending A/B) + +Deleted the 15 per-kind `if` blocks (`Columns, Melt, Dcast, Cov, Corr, +Transpose`, materializing `Join`, `Matmul, Update, Resample, Window, AsTimeframe, +Model, Construct/Stream, Program`) so they fall through to the single +`interpret_node` + `make_table_source` tail. Each was a hand-synced copy of an +`interpret_node` branch that produces the same table via the same table fn with +the same args and error strings (verified, including the grouped-update +rank/tuple dispatch and the `window` + `select_only` projection). `Model` +threads `model_out`; `Program` runs the preamble — both handled by +`interpret_node`. `build_binary_materializing_operator` had no other caller and +was removed. + +Initial wholesale collapse (route everything to `interpret_node`) confirmed a +regression: `join_filter_rank` +14.7% (`regression` verdict, 15 repeats). Root +cause — a `Filter` feeding a bucket-A breaker (here the grouped-rank `update` +between a join and its output filter) lost the fused parallel scan `build_operator` +gave it; `interpret_node` re-evaluated it whole-table and serial. + +### Step 3 — `build_materialized_fallback` keeps input construction on the physical path + +`build_operator_impl`'s tail now calls `build_materialized_fallback`, which: + +1. Resolves the node's **relational inputs** via `fallback_relational_inputs` — a + `switch` allowlist. Most kinds: the direct children. `Window`: the *grandchild* + (its direct child is an `update` clause `interpret_node` must own). `Stream`, + `Construct`, `Program`, and anything unlisted: none (their children are + template / expression nodes, not relational subtrees). +2. Builds and drains each input via `materialize_row_local` (= `build_operator` + + `materialize_operator`) — the fused parallel path. +3. Runs `interpret_node` over the node with those inputs handed back through a + new `ExecutionContext::pre_materialized_children` list (node ptr → table). + `interpret_node` checks it at entry and returns the pre-built table instead + of recursing; only direct inputs are listed, so the fallback node itself and + everything deeper are interpreted normally. + +`interpret_node` still owns every per-kind semantic; this only moves where the +inputs are built. Recoverable end state is unchanged — a kind can still be +lifted to a fully migrated breaker-over-pipeline, driven by +`physical_fallbacks_for(kind)`. + +Validation: all 1,815 non-slow tests; strict GCC; Debug + Release builds. A +first attempt without the `Window` grandchild / `Stream` exclusions failed 51 +tests (evaluating an `update` clause or a `__stream_input__` transform +standalone) — the allowlist is load-bearing. Release A/B over +`join,reshape,window,stats,transform,multi` vs pre-step-2 pending. + +### Step 3 — Separate bucket B from the fallback bucket + +`Scan`(deferred) and `ExternCall` are not materialized calls — the plan +deliberately leaves bare-source streaming to the executor (`EmptyChain`). Today +`note_materialized_call` fires for them, polluting the migration-backlog counter. +Either teach `plan_physical` to mark bare streaming sources `migrated`, or +exclude `EmptyChain` bare-source from the count. Keep the operators unchanged. + +### Step 4 — Prove bucket C dead + +A `Filter`/`Project`/`Rename` root over any classifiable source or breaker +becomes a migrated `MapPipeline`. Confirm the residual branches are reachable +only on `MalformedMapNode` (where they return the same structural error), then +replace with `invariant_violation` or delete. + +### Step 5 — Confirm recursion is gone + +After step 2 the only `build_operator` recursion left is inside migrated builders +(join / aggregate / order children) and bucket B sources. That is the +"remove obsolete `build_operator` recursion" goal met. + +### Step 6 — Validate + +Focused physical/interpreter tests; `explain physical` snapshot updates; full +non-slow suite (1,815); strict GCC runtime build; Debug + Release Parquet / +LightGBM plugin builds; `check-object-equivalence.sh` on the touched breakers; +Release interleaved A/B. + +## Expected outcome + +`chunked.cpp` drops ~400 lines. The fallback becomes one tail adapter plus a +`MaterializedCall` node that names its subtree in `explain physical`. +Mutation-tested `MaterializedCall` coverage in `tests/test_physical_plan.cpp` is +preserved and extended with the node name. + +## Not in scope + +Porting the 6 materializing joins (intended `MaterializedCall`); a join +build-side cost model; deferred-probe selectivity. Same exclusions as the parent +plan. diff --git a/plans/pipelined-execution-plan.md b/plans/pipelined-execution-plan.md deleted file mode 100644 index 0bdd8600..00000000 --- a/plans/pipelined-execution-plan.md +++ /dev/null @@ -1,641 +0,0 @@ -# Pipelined Execution - -Status: **partially implemented.** Multi-chunk correctness, streamed scans, -concurrent scan-unit decoding, the first source-to-map pipeline, and a bounded -join-probe handoff have landed. The general scheduler across arbitrary pipeline -breakers remains open. -Written 2026-08-14 from the thread sweep below; status updated 2026-08-16. -Read `plans/runtime-multithreading-plan.md` and -the removed `plans/chunked-execution-plan.md` first (git history); this proposes the thing -stop short of, and it supersedes neither. - -## The measurement that motivates it - -PDS-H SF-2, one box, `taskset -c 0-(N-1)`, the same core budget handed to both -engines, four archived runs in `benchmarking/tpch/results/runs/`: - -| cores | Ibex total | speedup | efficiency | Polars total | speedup | efficiency | Polars/Ibex | -|---|---|---|---|---|---|---|---| -| 1 | 3779 ms | 1.00× | 100% | 8164 ms | 1.00× | 100% | **2.16** | -| 2 | 2982 | 1.28× | 64% | 4141 | 1.87× | 93% | 1.39 | -| 4 | 2575 | 1.49× | 37% | 2671 | 2.73× | 68% | 1.04 | -| 8 | 2309 | 1.66× | 21% | 2000 | 3.44× | 43% | 0.87 | - -**Ibex is 2.16× faster than Polars on one core and 1.15× slower on eight.** - -The shape matters more than the endpoints. Ibex's *first* doubling returns -1.28×, against Polars' 1.87×. It is short from the start and short by roughly -the same proportion at every step (marginal +0.28 / +0.21 / +0.17 against -+0.87 / +0.86 / +0.71). That rules out contention and memory bandwidth, which -would show a near-ideal first doubling degrading later. - -Amdahl fits the whole curve from one number. Solving the 2-core point gives a -parallel fraction of **≈44%**; feeding that forward predicts 1.62× at 8 cores -against 1.66× observed. Ibex behaves like a program that is **56% serial**, at -every thread count. - -Per query the implied parallel fraction has a **median of 41%, and 14 of 22 -queries are below 50%**. The top of the range is fine — q06 at 93% scales -3.29×, beating Polars' own 2.09× on that query, and q01 at 82% reaches 2.71×. -Those are the queries that are essentially one row-local pass. Everything with -a join or a group-by is in the bottom half. - -This also explains why five separate operator-level threading attempts on -2026-08-14 each did what they claimed to their operator and moved the query by -0–3%: on a query that is 80% serial, doubling one operator's parallel phase is -worth almost nothing. Four were reverted for being unmeasurable and one for -regressing. - -## Diagnosis: materialize-then-fan-out - -The executor is a pull-based chunk pipeline already — `Operator::next()` returns -a `Chunk`, and `Chunk` even carries `sequence` and `row_offset` so an ordered -merger can reassemble morsels produced out of order. The substrate anticipated -this work. - -What is missing is that **nothing ever produces more than one chunk, and no two -chunks are ever in flight at once.** Three places enforce that: - -1. **Every production source is drained to one table.** The REPL's batch path - resolves a reader through `lazy_table_func`, calls `LazyTable::project`, and - wraps the result in `TableSourceOperator`, which emits exactly one chunk. - Verified: `chunks=1` on every operator of every PDS-H query. -2. **A chunked source is materialized even when one exists.** - `ChunkedParquetSourceOperator` streams 65536-row Arrow batches - (`plans/bigger-than-ram-plan.md` Phase 4), and `src/runtime/chunked.cpp` - around the `chunked_table_func` branch immediately calls - `materialize_operator()` on it. -3. **Parallel islands materialize before fanning out**, by a documented - load-bearing invariant: the island's input subtree is executed to a `Table` - on the calling thread, and every morsel source below takes that finished - table by reference. That is what makes a `LazyTable` safe inside an island. - Islands also only cover row-local chains, which is why only 5 of 22 queries - form one at all. - -So each operator runs to completion, on one thread, before the next begins. -Whatever parallelism exists is strictly *intra*-operator with a serial merge -between, and the serial phases of every operator add up with nothing to overlap -them. That is the 56%. - -Polars reaches 3.44× because morsels flow through the whole pipeline -concurrently: one morsel joining while another decodes and a third aggregates. - -A consequence worth stating because it is a correctness risk, not just a -performance one: **the cross-chunk paths in the operators have never run.** -`KeyPartition::stored`, `partitioned_active_`, `cat_dictionary_id_`, the -distinct operator's `packed_part_count_` pinning, the pair path's dense-array -rebuild — all of it is written for multi-chunk input and all of it is dead -today. One latent bug in that machinery was already found and fixed on -2026-08-14 (`4ba4b75`) purely by reading. There will be others. - -## The constraint that shapes the design - -The single-chunk path is not an accident or laziness — it is where the biggest -wins of the last months live, and they are worth more in absolute terms than -the threading gap: - -- projection pushdown (`plans/parquet-*`) -- dynamic filter pushdown, geomean ≈ −12% -- decode fusion incl. late materialization, geomean −14% -- null-free stats fast paths, geomean −27% -- the fused key-filter scan, q17 −28% - -Every one of those works by giving `LazyTable` the *whole* query's demand — -columns, conjuncts, join keys — and letting it decode once, minimally. A naive -"stream row groups through the pipeline instead" throws all of it away, and the -arithmetic is not close: −12/−14/−27% against a threading ceiling that, even if -perfectly achieved, is worth ~1.6× on 8 cores. - -**So the design constraint is: pipelining must be built on top of the pushdown -machinery, not instead of it.** The natural shape is that a scan still plans its -decode globally (all pushdowns intact) but *yields* the result in row-group or -batch units rather than as one table. - -## Phases - -Deliberately ordered so each is separately measurable and the risky one is last. - -### Phase 0 — make multi-chunk real, and prove it correct — **DONE (`f9db6a0`)** - -No performance goal. Get more than one chunk flowing and find out what breaks. - -- A test-only or env-gated switch that makes a source emit N chunks for a - materialized table (`PartitionedTableSource` already does exactly this, and is - currently used only by islands and tests). -- Run the full PDS-H answer check and the 1574-test suite with it on, at several - grains including pathological ones (1 row, 1 chunk, prime-sized). -- Extend the parity comparator (`plans/done/serial-parity-comparator-plan.md`) to - assert chunked-vs-single-chunk equality structurally, not by diffing stdout. - -Exit criterion: every query is byte-identical at every grain. Expect real bugs -here — this machinery has never executed. - -### Phase 1 — a streaming scan that keeps its pushdowns — **DONE, on by default** - -- Give `LazyTable` a way to yield its planned decode in units (row group, or - Arrow batch) instead of one table, with projection, conjuncts, dynamic key - filters and late materialization already applied. -- Route the batch path to it, keeping `TableSourceOperator` for anything that - cannot. -- Measure with pushdowns on. The bar is *no regression*: this phase buys memory - and cache locality, not parallelism. - -Trap: `LazyTable::cache_` is not thread-safe and the deferred-probe path -explicitly declines when a key column is already cached. Streaming must not -quietly disable the fused scans — check by diffing plan shapes (profiler -`op="..."` counts), which is how a silent decline was caught on 2026-08-14. - -**What landed.** `SourceUnit` is a source-global row range a reader can decode -alone; `LazySourceReader::decode_units()` reports them (Parquet: one per row -group) and `decode` / `key_filter_scan` / `string_filter_scan` all take one. -`LazyTable::project_where_unit` is `project_where` restricted to a unit, with -every pushdown applied to that unit rather than declined — the fused scans -included, because both already plan over the whole file and answer in -source-global indices, so restricting them is a filter on their group list. -`DeferredScanSourceOperator` drives it. **Streaming is the default**; -`IBEX_STREAM_SCAN=0` opts out, and answers both ways for the same reason -`IBEX_PARALLEL` does — a switch that could only turn it on leaves no way to -turn it off. - -Two things had to be got right and are worth remembering. Selections stay -source-global at every boundary, which is what lets the unit path reuse the -whole-source filtering code instead of growing a second index space. And a unit -decode never touches `cache_`: a unit holds a *fragment* of a column, and a -fragment in the cache is indistinguishable from a whole one to every later -reader — including the fused scans, which decline when their key column is -cached. - -**The bar was not met, and the way it was missed is the useful result.** -PDS-H SF-1, interleaved, min-of-5, answers identical and plan shapes identical -(no pushdown silently declined): - -| | geomean vs materialized | -|---|---| -| 1 core | **0.943** | -| 8 cores, `taskset -c 0-7` | **1.084** | - -Streaming is **5.7% faster on one core and 8.4% slower on eight**. Since the -same binary does the same work in both, the regression is not extra work — it -is *lost parallelism*, and the single-core number says the phase delivered -exactly the cache-locality win it promised. q01's decode confirms it directly: -pool work drops 234ms → 125ms while wall rises 123ms → 208ms, and occupancy -falls 0.44 → 0.14. - -The cause is the diagnosis above, one level down. A unit is ~1M rows, which is -plenty to parallelize (`parallel_min_rows` is 65536, so no gate is being -missed) — but six units run one after another with a serial phase between each, -so the pool sees six short bursts instead of one long one and nothing overlaps -them. Materialize-then-fan-out, at unit granularity. - -**This was Phase 2's case, made quantitatively.** Concurrent scan units are -not a refinement of Phase 1; they turn the 0.943 into a better 8-core number. -They do *not*, by themselves, turn the operator chain into a pipeline. The -remaining scheduler work is what must move the parallel fraction. - -Per-query, the 8-core split is wide (q12 −29%, q08 −9%; q01 +69%, q20 +42%, -q19 +36%). q21 is the one query that loses single-threaded too (+13.6%) and is -the place to start if Phase 2 lands and something still regresses. - -### Phase 2 — concurrent chunks — **scan slice and join slices landed; scheduler open** - -The desired end state is multiple chunks in flight through the operator chain. -What has landed so far is narrower: multiple scan units decode concurrently, -then their window is harvested and consumed in order. This is useful -intra-source parallelism, but it is not pipeline concurrency. - -**What landed: concurrent units inside the scan.** The scan decodes a WINDOW of -units on worker threads instead of one after another. Ordering is untouched — -workers claim units from a shared cursor and write only their own slot; after -the caller harvests the completed window, chunks are served in unit order with -`sequence` / `row_offset` assigned on the calling thread. - -This distinction is load-bearing: the source waits for the entire window before -it returns its first chunk, and it cannot start the next window until the -current one has been consumed. Decode can use the pool, but downstream joins, -aggregates, and row-local operators do not overlap with it. Treat any result -from this slice as scan parallelism, not evidence of end-to-end scaling. - -Decoding a unit on a worker is safe because `LazyTable::acquire_reader` hands -each concurrent acquisition its own reader product (that is what the reader pool -was built for), `project_where_unit` never writes `cache_`, and every inner -parallel path checks `on_worker_pool_thread()` and runs serial inside a task. -The middle one is now load-bearing rather than merely tidy: routing any part of -the unit path back through `project()`, which does cache, turns it into a race. - -PDS-H SF-1, 8 cores, interleaved, geomean against the materialized path: - -| | geomean | -|---|---| -| Phase 1 (units, serial) | 1.084 | -| Phase 2 (units, concurrent) | **0.922** | - -Streaming is now **7.8% faster** than materializing, where Phase 1 was 8.4% -slower. q12 −44%, q06 −31%, q04 −23%, q19 −21%, q14 −19%, q01 −17%, q15 −16%. - -**Two fixes mattered more than the concurrency itself**, and both were found by -following the profile rather than by reasoning about the design: - -* *Per-row dictionary interning.* The chunk-to-chunk categorical remap interned - **per row** instead of per dictionary entry. On q01 that was 114ms of a 160ms - scan — the entire regression — and it was invisible from the query: TPC-H's - `l_returnflag`/`l_linestatus` are plain `string` in the Arrow schema and only - become Categorical because the writer dictionary-encoded them. Interning each - dictionary entry once and gathering codes took q01 from +90% to −18%. -* *A concat that only existed because chunks did.* The semi/anti join's swapped - path called `MaterializeOperator` on its left, which was free when the left - was one chunk and a full copy once it was six. It buffers the chunks as a list - now — it needs the left twice, but never glued. q21's semi join went 113ms -> - 87ms, and the query from +40% to +10%. - -The second is the shape to expect more of: **operators that were written against -a one-chunk world hide a concat.** They are correct either way, so only a -profile finds them. - -**q21, run down.** The residual was in a profiled statement after all — an -earlier single-sample reading said otherwise and was wrong. Statement 1 (+16%) -carried the whole of it, and inside that statement the cost was not any operator -but the **sink**: `MaterializeOperator` spent 33ms appending where the -single-chunk path spent 0, because one chunk is *moved* into the result table -and six must be concatenated. Appending was a `push_back` per row even for a -flat numeric column; `append_column_values` now bulk-copies, which took -statement 1 from +16.4% to +8.9%. What is left is the memmove itself — -`li_F` is 2.9M rows of four POD columns, so ~58MB of copying that the -single-chunk path never does. - -That cost is inherent while a `let` binding is one contiguous `Table`: streaming -a large intermediate into a binding must glue it back together. It is the same -lesson as `plans/runtime-multithreading-plan.md`'s "the MERGE CONCAT is the real -island cost", and it says the wins track *output* size — which is exactly the -observed split, since every query that gains reduces its rows sharply (q06, -q12) and every one that loses binds a large intermediate. - -**q20, run down — and the general lesson.** Its high-cardinality group-by cost -50ms -> 79ms under streaming, and the cause was not the aggregate's merge but -its **gate**: `pool_tasks` went 32 -> 0, i.e. partitioned group discovery -declined outright. The gate asks "are there enough rows here to be worth -partitioning", and it asked it of the CURRENT CALL. q20's aggregate sees 909k -rows over 543k groups — comfortably qualifying — but as six chunks that is -~151k per call, under the 262144 threshold, so it declined on every one. - -**Chunking divides every per-chunk row gate by the number of chunks.** That is -the systemic consequence of this phase and it will keep biting; q20 is simply -where it bit first. The engine's other row gates use `parallel_min_rows` -(65536) and still clear it at these sizes, which is why only this one showed. - -Fixed by counting the rows the *operator* has been offered rather than the rows -in this call. The threshold itself is unchanged — lowering it is a measured -dead end, because the break-even is set by group cardinality, not row count -(`plans/` history: q13 +9.3%). Starting part-way through a stream then means -groups already exist in the serial index, which the partitioned path neither -reads nor writes, so they are seeded into the partitions keeping their existing -ids. Only the packed path cannot do this — its key is built from a row and is -not invertible — so it declines to start late, which is the previous behaviour. - -q20 +23% -> +10%, q21 +9% -> +5%, suite geomean unchanged at 0.921. What is -left on q20 is that its first chunk is still discovered serially and its groups -then seeded; closing that needs the aggregate to defer its first chunk until -the decision is made. - -**q22 +7%** and the rest of **q21** are the concat above. The scan itself is -now faster than materializing on every query measured. - -### Turning it on by default - -Flipping it found a **data-loss bug** that every measurement up to that point -had missed: the window loop asked `batch_.has_value()` to decide whether a -window was in flight, and a ONE-UNIT window is decoded inline and never -submits. So a serial query dropped every unit after the first, and a parallel -one dropped any trailing single-unit window. The PDS-H suite never saw it — -lineitem's 6 units fit one window of 8 exactly — and only the e2e's -`IBEX_PARALLEL=0` leg, which asserts a row total, caught it. The e2e streaming -check now runs serially on purpose rather than by luck. - -The lesson: **the shapes that break streaming are window remainders and serial -execution**, and neither is exercised by the benchmark suite. - -### Memory: not the free win it looks like - -Streaming was expected to bound peak memory. It does — but only below the -window. Peak RSS scales with the window, which is the thread budget: - -| PDS-H SF-1 | materialized | streamed w1 | streamed w8 | -|---|---|---|---| -| q04 | 193M | 127M | 268M | -| q19 | 137M | 57M | 170M | - -So `IBEX_THREADS` now bounds peak decode memory as well as parallelism, and at -the default window several queries use 20-47% MORE than materializing. Much of -that excess is not live data: it is glibc growing a free list per worker arena, -since decoding moved onto the pool. `MALLOC_ARENA_MAX=1` takes q13's +33% down -to +2% and q04's +45% to +20%. - -A lookahead window — dispatching window k+1 before serving window k — was built -and **reverted as a measured dead end**: +52% peak RSS on a 25-row-group scan -(161MB -> 244MB) for zero wall-clock change, because the consumer is a blocking -operator that eats chunks faster than they decode. There is no consumer work to -overlap with until the rest of the pipeline runs concurrently. - -### What the sweep says about the rest of the phase — read this first - -Repeating the thread sweep was this plan's own gate ("the number that must move -is the implied parallel fraction, not the wall time"). Run at SF-2 on the same -harness, materialized vs streamed: - -| | 1c | 2c | 4c | 8c | speedup | implied parallel fraction | -|---|---|---|---|---|---|---| -| materialized | 4990 ms | 4153 | 3568 | 3236 | 1.54× | 34% | -| streamed | 4456 ms | 3967 | 3365 | 3033 | 1.47× | **22%** | - -Streaming is faster at every core count (−11% at 1c, −6% at 8c) and the -parallel fraction went **DOWN**. It did not make more of the program parallel; -it made the serial part cheaper. **The premise at the top of this document — -that pipelining is what raises the parallel fraction — is not what Phase 2's -first slice delivered**, and nothing about the rest of the phase should be -justified by it without new evidence. - -(Caveat: this harness times whole processes, so startup and plugin load are -counted as serial and deflate the fraction against the `run_bench.sh` numbers -in the table at the top. The materialized-vs-streamed comparison is same-harness -and sound; the absolute 34%/22% are not comparable to the 44% above.) - -### Where the serial time actually is - -Calling-thread ms summed over real operator nodes across the suite, 8 cores, -with the worker help each drew (`pool_work / self`). The first column is the -reading that opened this section; the second is the same measurement after the -joins were worked on. - -| operator | self ms (before) | self ms (now) | pool ms (now) | worker help | -|---|---|---|---|---| -| scan | 479 | 631 | 2902 | 4.6× | -| aggregate | 473 | 552 | 945 | 1.7× | -| **join inner** | **422** | **435** | 80 | **0.2×** | -| join semi | 326 | **183** | 285 | 1.6× | -| join anti | 33 | 36 | 14 | 0.4× | -| update | 57 | 70 | 145 | 2.1× | -| distinct | 40 | 48 | 191 | 3.9× | - -**Joins were 42% of calling-thread operator time and drew essentially no worker -help.** That reading was right, and it is what the rest of this phase should be -steered by. The scan, which this phase spent its effort on, is now the -best-parallelized operator in the engine. - -**How to read `self ms` — settled by reading the profiler, after getting it -wrong twice.** `ExecutionProfileScope` pushes a frame per scope and adds its -whole elapsed time to its parent's `child_ns`, which the parent subtracts. That -applies to `ProfilePhase::Source` exactly as it does to a nested operator, so -**a source stage's time is EXCLUDED from the enclosing operator's self, not -added to it**. A mid-session claim that `join inner`'s 422ms was mostly its -deferred probe's decode was wrong on that mechanism; the empirical check is q03, -whose source stages cost 147ms of calling-thread time while its two `join inner` -rows report 10.6ms and 7.6ms between them. - -The practical consequence is the opposite of what that wrong reading implied: -the joins' self ms is real join work, and the ranking above stands. - -1. **Semi/anti join** — **DONE.** Its `filter_chunk` was 266ms across the suite - with the pool idle, split 4:1 predicate to gather. Only the predicate was - threaded: each one probes a set that stops changing before the first left - chunk arrives, so ranges need no coordination and each builds its own index - list. The gather is deliberately left serial — threading a gather here is the - same memory-bound dead end already recorded over `ChunkedDistinctOperator`. - q21 −29%, suite −4.6%; the row above went 326 self / 0 pool to 183 / 285. -2. **Inner join** — still 435ms at 0.2×, but **it is not one lever**. Timed - directly, the operator's own work splits into `probe_chunk` 91ms, - `assemble_output` 48ms, `build_index` 42ms, `emit_swapped` 7ms, and a - remainder that is phase A's IN-MEMORY passes (`filter_selection`, - `apply_membership_filter`, the two-phase hit loop) — those are not source - stages, so they land in the join's self. - - So the shelved `git stash` "parallel inner-join probe" addresses - `probe_chunk` + `assemble_output` ≈ **139ms**, or 3-4% of suite wall if - perfectly parallelized. Worth doing, but it is not what makes the join row - large, and it should not be sold as such. - - Also measured: the full two-phase branch's `left_copy` — a deep copy of the - build table — **never runs in PDS-H**. `build.rows() > sel.selected.size()` - sends every query down the `RightMaterialized` branch instead. Do not spend - effort there without a query that reaches it. - -### What came out of following that ranking - -Two changes landed that the ranking did not predict, both found by timing phase -A of the deferred probe rather than the operator that owns it: - -* **Phase A now fuses membership past a static filter.** Its fused key scan - required `conjuncts.empty()`, a condition copied from `project_where`, so a - probe scan carrying a filter decoded the whole key column AND the predicate - column and then walked every row twice, serially. Membership now runs inside - the decoder and the conjuncts are evaluated through its selection. q03 −37%, - q07 −24% at 8 cores. Gated on fixed-width conjunct columns: a sparse read of a - variable-width column is not proportionally cheaper (q10 +9.3% when it was - not gated). -* **The key scan's abandon rule is asked 16× sooner.** It needs 262k rows to - fire but was only asked at ~1M-row group boundaries, so a doomed scan decoded - a whole group — and in parallel every other in-flight group finished - alongside it. Group 0 now asks it per 64k batch, which is sound because group - 0's rows ARE the file-order prefix. That removed the q05/q10 regressions the - fusion change introduced, and those regressions had SCALED WITH CORE COUNT - because the overshoot was bounded by one worker wave. - -Cumulative for both, PDS-H SF-2, interleaved, min of 6: suite −3.3% at 1 core, -−3.4% at 4, −4.0% at 8, with q03 −37% and q07 −24% at 8. - -**Still open where phase A is concerned:** membership-first wins only when -membership is more selective than the conjuncts, and nothing at this layer -estimates that. q10 probes `orders` against an unfiltered customer table and its -scan still abandons — now cheaply. The estimate would come from the footer-stats -cost model behind `join_reorder`. - -The scheduler remains the design for running the whole chain concurrently. The -original thread sweep does argue for it: the current scan slice makes the -single-core path faster but leaves Ibex at 1.66x on eight cores, versus Polars' -3.44x. The next phase must be judged by whether it raises the implied parallel -fraction and reaches at least Polars parity at the fixed core budget, not by a -small operator-local win. - -**What landed: the first source pipeline.** A decomposable lazy scan now -publishes units through a bounded ordered ring as soon as they complete; it no -longer waits for a whole decode window. A downstream breaker therefore consumes -earlier chunks while later units are still decoding. When a maximal row-local -chain starts directly at the scan, `build_operator` builds one private chain per -worker and the same task immediately runs its decoded unit through those maps -before publishing it. There is no whole table or whole-window wait at either -boundary. Empty-schema carriers, categorical dictionary unification, sequence -order, cancellation, and backpressure are preserved at the ordered publication -point. - -One pool thread is reserved for downstream parallel work when a long source can -fill the ring and park every producer. Without that reservation a blocking -operator can submit a batch while every pool thread is on the backpressure -condition: the caller waits for the batch, the scan workers wait for the caller -to drain, and neither can move. A source of at most `3 * workers` units cannot -reach that state — the ring holds `2 * workers` and the workers have already -claimed at most one unit each — so those common short Parquet scans retain the -full decode budget. A one-thread pool keeps the old serial-window source. - -This is deliberately the first scheduler slice, not the completed scheduler. -It covers source → optional row-local maps → breaker, including the direct -scan-to-aggregate shape and the map chain that previously hit -`build_parallel_island`'s materialize-before-fan-out boundary. - -**What landed next: a bounded join-probe handoff.** `PipelinedStageOperator` -keeps the pull `Operator::next()` API on both sides of a breaker while a -dedicated producer thread drives the join below it and publishes at most two -ordered chunks. Its caller can consequently aggregate or map one probe output -while the join pulls and probes the next. It is intentionally not a -`WorkerPool` task: a streamed scan already owns tasks from that fixed pool, and -using the same pool for a long-lived stage recreates the producer/backpressure -deadlock that the scan worker reservation avoids. Cancellation wakes both queue -waits before joining; producer errors cross the queue exactly once. - -The builder stages only streamable inner/semi/anti joins whose *left/probe* -subtree contains a multi-unit deferred scan. One-chunk table joins have no -overlap to expose and stay on the original pull path; terminal aggregates also -stay on the caller because a hash aggregate emits only after its input ends. -`ParallelIslandStats::pipelined_stages` and the deferred four-unit join test -make both the gate and ordering observable. - -This is the breaker-output mechanism Phase 2 needed, but it is not yet a -general scheduler: aggregate outputs, other semi-blocking probe paths, shared -stage admission, and retirement of the materialized-island executor remain -open. The stage is also not a scaling win yet. A same-binary release screening -run at SF-1 (five timed iterations, not interleaved) measured 4 cores -1208→1255 ms (+3.9%, query geomean 1.042) and 8 cores 1070→1075 ms (+0.5%, -geomean 1.018). Treat that as a negative result, not a benchmark claim: the -general scheduler needs progress-aware admission and a repeated interleaved -sweep before this stage can satisfy Phase 2's performance gate. - -PDS-H SF-1, pinned cores, whole-script mode, release builds at `8f1e349` -versus this slice (warmup 1-2, 5-7 timed iterations): - -| cores | baseline total | pipeline total | total delta | query geomean | -|---|---:|---:|---:|---:| -| 1 | 1976 ms | 1913 ms | -3.2% | 0.963 | -| 2 | 1583 ms | 1542 ms | -2.5% | 0.963 | -| 4 | 1355 ms | 1292 ms | -4.6% | 0.935 | -| 8 | 1141 ms | 1092 ms | -4.3% | 0.964 | - -The implied parallel fraction moves only about one percentage point (48% to -49% from the 1/8 totals), so this is a useful source-overlap win, not yet the -curve-changing scheduler the phase is aiming for. q01 is consistently about -14% faster at eight cores; q22 is the consistent outlier at +12-19%. The -phase's original no-regression bar was explicitly loosened for this slice on -2026-08-16, so q22 remains follow-up work rather than a reason to keep the -pipeline disabled. - -SF-4 makes both the scaling benefit and a low-core scheduling defect clearer. -The table below is the mean of two order-balanced sweeps of the same baseline -and worktree, with one warmup and five timed iterations per query in each -sweep. Higher scale factors are timing-only because the official qualification -answers in this repository apply only to SF-1. - -| cores | baseline total | pipeline total | total delta | query geomean | -|---|---:|---:|---:|---:| -| 1 | 8305 ms | 8419 ms | +1.4% | 1.000 | -| 2 | 6309 ms | 6742 ms | +6.9% | 1.077 | -| 4 | 5274 ms | 5242 ms | -0.6% | 0.951 | -| 8 | 4533 ms | 4366 ms | -3.7% | 0.953 | - -The one-core geomean is neutral, as expected because the scan pipeline is -disabled for a one-thread pool; its total is dominated by noisy long-tail -queries and is a control rather than evidence of pipeline overhead. At four -and eight cores the broad query geomean improves about 4.7-4.9%, and the 1/8 -totals move the implied parallel fraction from about 52% to 55%. q18 remains -the large counterexample (+24% at four cores, +13% at eight); q20 is +10% at -four cores and q17 is +12% at eight. - -Two cores expose a separate policy cliff. Long sources reserve one of the two -pool threads to avoid producer/breaker deadlock, leaving a single decode -producer while still paying the pipeline handoff cost. The suite is +6.9%, -with q06 +47%, q04 +24%, and q12 +24%. A follow-up admission gate that declined -the one-producer configuration was remeasured against `6ef0c60` at SF-4 and -made the important late queries materially worse: q19 +28%, q20 +21%, q21 -+20%, and q22 +17% in a target-then-baseline three-iteration sweep. The -producer still overlaps decode with the caller's serial breaker work, so the -gate was withdrawn. The next scheduler slice must replace static reservation -with progress-aware admission/backpressure rather than infer usefulness from -the producer count. The relaxed no-regression bar permits keeping this first -slice enabled, but the two-core crossover and q18 are named follow-up gates -rather than noise to average away. - -The rest of this phase must retain the same bounded-queue contract: a source -may publish completed units as they become ready, row-local stages may consume -and publish morsels independently, and a blocking stage supplies backpressure -before the next pipeline. A whole-window `wait()` at the source boundary is -expressly not sufficient. - -Requires deciding, per operator, which of three it is: - -| class | operators | behaviour | -|---|---|---| -| streaming | filter, project, rename, update (row-local) | chunk in, chunk out; trivially concurrent | -| blocking | aggregate, sort, distinct, join build side | must see all input before emitting | -| semi-blocking | join probe, top-k | blocked on build, streaming on probe | - -The classic morsel-driven answer is that a *pipeline* runs from a source to the -next blocking operator, and pipelines are scheduled with the blocking operator -as a barrier. That is a scheduler, and it replaces `build_operator`'s -straight-line chain — this is the large part of the work and should not be -started before Phases 0 and 1 have landed and held. - -Ordering is the contract to preserve: `Chunk::sequence` / `row_offset` exist for -exactly this, and `MaterializeOperator`'s in-order concat is the existing -ordered merger. Everything Ibex reports in first-occurrence order (group-by -output, distinct, `head`) depends on it. - -### Phase 3 — retire the island special case - -If Phase 2 lands, parallel islands become a special case of the general -scheduler and the materialize-before-fan-out invariant can go. Not before. - -## Validation - -- **Answers**: all 22 PDS-H answers byte-identical at every phase, at every - grain. Non-negotiable; this is the gate that catches the dead cross-chunk - paths waking up. -- **Suite**: `ctest` (1574) plus `scripts/ibex-e2e.sh` at each phase. -- **Performance**: `run_bench.sh` archives + `compare_runs.py`, which reports a - reference engine's drift so a box-condition change cannot be read as a result. - Repeat the 1/2/4/8 sweep at each phase. The scheduler's acceptance gate is - **at least Ibex/Polars multi-core parity at the same pinned core budget**; - the leading diagnostic is the **implied parallel fraction**, not a - single-core or operator-local wall-time change. -- **Interleaved A/B with a control query** for anything narrower, per the - methodology that caught three false positives on 2026-08-14. - -## Non-goals - -- **Not out-of-core.** `plans/bigger-than-ram-plan.md` owns spilling. Streaming - helps peak RSS as a side effect; that is not the goal here and must not be - used to justify a regression on the timed path. -- **Not a rewrite of the operators.** They are already chunk-shaped. The work is - in what feeds them and what schedules them. -- **Not more intra-operator threading.** That is the thing this measurement says - has run out of road. `plans/runtime-multithreading-plan.md` Phase 4 remains - open for specific gaps, but it is not the answer to the curve above. - -## Open questions to settle before the scheduler slice - -1. Where does the scheduler live — inside `build_operator`'s seam (the - `execution-plan-seam-plan.md` Option B position), or above it? -2. What is the chunk grain, and is it per-source or global? `IBEX_MORSEL_ROWS` - exists for islands; a pipeline may want a different answer. -3. How does a blocking operator's *build* phase get parallelised, or does it - stay serial? The measurement says `build_index` is only 1.5% of q10, so - possibly it stays serial forever, which would simplify the scheduler a lot. -4. Does the REPL's statement-at-a-time model need to change, or can a pipeline - stay inside one statement? Everything above assumes the latter. - -## Honest assessment of size - -Phase 0 is days and will surface bugs. Phase 1 is a substantial change to the -most performance-sensitive code in the tree, with a hard no-regression bar -against four separate pushdown mechanisms. Phase 2 is a scheduler and is the -kind of change that touches the contract every operator is written against. - -The upside is bounded and known: the curve says a perfect result is ~3.4× on 8 -cores where we get 1.66×, i.e. roughly halving total PDS-H time and turning a -0.87 ratio into ~1.5 in Ibex's favour. That is the largest single number -available anywhere in the tree, and it is not reachable in pieces — which is the -argument for doing it, and equally the argument for scoping it properly first. diff --git a/plans/runtime-multithreading-plan.md b/plans/runtime-multithreading-plan.md index 08e8fe90..55da8e76 100644 --- a/plans/runtime-multithreading-plan.md +++ b/plans/runtime-multithreading-plan.md @@ -5,7 +5,8 @@ development diary and the completed-phase detail moved to git history at the pre-compaction commit's parent. `parallelism-overview.md` is the current map of what actually shipped; this file is the phase roadmap and the two design sections that are still load-bearing (the LazyTable Synchronization Contract, -Phase 2 RNG). +Phase 2 RNG). The separate pipelined-execution plan was removed as superseded; +its remaining scheduler work is owned by `kernel-pipeline-execution-plan.md`. > **Nomenclature drift:** this plan predates the config rename. `IBEX_THREADS` → > **`IBEX_CORES`** (compute budget); `IBEX_PARALLEL` was **removed** — serial is @@ -16,10 +17,10 @@ Phase 2 RNG). Multithreading as a query-execution capability, not ad-hoc loops inside kernels. The `Chunk` operators are the data unit for a morsel-driven executor. A query is -a sequence of **parallel islands** (maximal runs of row-local, chunk-preserving -operators) separated by **barriers** (order / join / group-by / distinct / -window / rank / model). Start with one ordered parallel island for row-local -work, keep serial implementations behind barriers, expand only after each +a sequence of **morsel-parallel map pipelines** (maximal runs of row-local, +chunk-preserving operators) separated by **barriers** (order / join / group-by / +distinct / window / rank / model). Start with one ordered parallel map pipeline +for row-local work, keep serial implementations behind barriers, expand only after each operator family has an explicit correctness contract. DuckDB's model (partitionable sources, pipeline breakers, worker-local + sparing query-global state) without a general DAG scheduler. @@ -87,14 +88,14 @@ documented ownership contract — **not** blanket "make it thread-safe". The hazard surface is narrower than the object. **Interim gate (Phase 0) — LIFTED 2026-08-02, worth ~nothing.** The gate made -any query reading a lazy source island-ineligible. Removed once established that -no worker can reach a `LazyTable`: `build_parallel_island` materializes its -input subtree to an owned `Table` on the building thread, and every morsel -source takes that finished table by `const Table&`. Measured: 2 of 22 PDS-H -queries changed eligibility, q19 gained an island (no time change), **only 5 of -22 form an island at all** (q18/q19/q21×2/q22). The gate was never what kept +any query reading a lazy source map-pipeline-ineligible. Removed once +established that no worker can reach a `LazyTable`: the morsel pipeline +materializes its input subtree to an owned `Table` on the building thread, and +every morsel source takes that finished table by `const Table&`. Measured: 2 of +22 PDS-H queries changed eligibility, q19 gained a map pipeline (no time +change), **only 5 of 22 form one at all** (q18/q19/q21×2/q22). The gate was never what kept PDS-H serial — whole-script mode eagerly projects non-probe scans, and scan -conjuncts get pushed into the decoder (removing the `Filter` an island builds +conjuncts get pushed into the decoder (removing the `Filter` a map pipeline builds from). What's left above the scans is joins/group-by/sort — barriers. **The PDS-H multithreading gap is Phase 4, not Phase 3b.** A slice that streams a source's morsels straight into workers reintroduces the hazards and must @@ -181,13 +182,12 @@ actively detects re-entry** — a fake `ColumnDecodeFn` that sets an atomic in-flight flag on entry and fails if already set. ThreadSanitizer build for the concurrent-lazy-scan tests. -## Phase 1 — First parallel island — LANDED, ON by default +## Phase 1 — First morsel-parallel map pipeline — LANDED, ON by default `ExecutionContext::parallel` defaults true. What shipped, in order: -- **Serial island + worker pool + ordered merger** — `ParallelIslandOperator` +- **Morsel pipeline + worker pool + ordered merger** — `MorselPipelineOperator` dispenses morsels off one atomic cursor, per-worker map chain, bounded ring - merge by `sequence`. `SerialIslandOrderValidator` asserts the two executors - are byte-identical. + merge by `sequence`. Tests assert serial and parallel byte-identity. - **Row-local `Update` eligibility** — an unguarded, ungrouped, tuple-free update whose every field is `is_subset_evaluable_expr` (stricter than `is_row_local_update_expr` — the looser one admits aggregates, which per @@ -284,8 +284,9 @@ ordered units; units decode concurrently with independent reader products; a direct source or maximal row-local chain publishes completed units downstream without waiting for a decode window. First bounded join-probe output handoff landed. Pushdown / cancellation / backpressure / dictionary unification -preserved. See `pipelined-execution-plan.md` for SF-1/SF-4 measurements and the -unresolved two-core admission problem. +preserved. The historical SF-1/SF-4 measurements are in git; the unresolved +two-core admission problem and general breaker work are now tracked by +`kernel-pipeline-execution-plan.md`. **Still open:** CSV, TSAN coverage, generalized source partitioning (SF-1 has too few row groups — must partition columns and row ranges, not just row diff --git a/src/ir/join_output.cpp b/src/ir/join_output.cpp index d49cc7d1..39d71974 100644 --- a/src/ir/join_output.cpp +++ b/src/ir/join_output.cpp @@ -161,4 +161,41 @@ auto plan_join_output(JoinKind kind, const std::vector& keys, return plan; } +auto resolve_join_columns(JoinKind kind, const std::vector& keys, + std::span left_names, + std::span right_names, + const JoinSuffixPolicy& suffix) + -> std::expected { + auto output = plan_join_output(kind, keys, left_names, right_names, suffix); + if (!output.has_value()) { + return std::unexpected(std::move(output.error())); + } + + JoinColumnMapping mapping; + mapping.left_input_names.reserve(left_names.size()); + for (const std::string_view name : left_names) { + mapping.left_input_names.emplace_back(name); + } + mapping.right_input_names.reserve(right_names.size()); + for (const std::string_view name : right_names) { + mapping.right_input_names.emplace_back(name); + } + mapping.output = std::move(*output); + mapping.keys.reserve(keys.size()); + for (const JoinKey& key : keys) { + const auto left = std::ranges::find(left_names, key.left); + if (left == left_names.end()) { + return std::unexpected("join key " + quote(key.left) + " not found in left input"); + } + const auto right = std::ranges::find(right_names, key.right); + if (right == right_names.end()) { + return std::unexpected("join key " + quote(key.right) + " not found in right input"); + } + mapping.keys.push_back( + {.left_index = static_cast(std::distance(left_names.begin(), left)), + .right_index = static_cast(std::distance(right_names.begin(), right))}); + } + return mapping; +} + } // namespace ibex::ir diff --git a/src/ir/schema.cpp b/src/ir/schema.cpp index 74ad73be..2a9fa5fd 100644 --- a/src/ir/schema.cpp +++ b/src/ir/schema.cpp @@ -687,19 +687,17 @@ auto check_one_join(const JoinNode& join, const SchemaInfo& left, const SchemaIn for (const auto& key : join.keys()) { const auto* left_field = left.find(key.left); const auto* right_field = right.find(key.right); - // Only a closed schema proves absence: an open one lists the columns it - // knows about and admits others. - if (left_field == nullptr && !left.is_open()) { + // Open schemas admit extra physical columns, but those columns are not + // addressable until an ascription names them. Every referenced join + // key therefore has to appear in the declared portion on both sides. + if (left_field == nullptr) { return "join key '" + key.left + "' not found on the left side (available: " + format_field_names(left) + ")"; } - if (right_field == nullptr && !right.is_open()) { + if (right_field == nullptr) { return "join key '" + key.right + "' not found on the right side (available: " + format_field_names(right) + ")"; } - if (left_field == nullptr || right_field == nullptr) { - continue; - } if (!left_field->type.has_value() || !right_field->type.has_value()) { continue; // an untyped column is still known to exist } diff --git a/src/repl/CMakeLists.txt b/src/repl/CMakeLists.txt index 3951c259..2dc30c2f 100644 --- a/src/repl/CMakeLists.txt +++ b/src/repl/CMakeLists.txt @@ -21,6 +21,8 @@ target_include_directories(ibex_repl PUBLIC $ $ + PRIVATE + ${PROJECT_SOURCE_DIR}/src ) target_link_libraries(ibex_repl diff --git a/src/repl/repl.cpp b/src/repl/repl.cpp index 8145c9b8..25c73eec 100644 --- a/src/repl/repl.cpp +++ b/src/repl/repl.cpp @@ -51,6 +51,8 @@ #include #include #include + +#include "runtime/physical_plan.hpp" #ifdef _WIN32 #define NOMINMAX #include @@ -140,7 +142,7 @@ enum class ReadLineStatus : std::uint8_t { Line, Eof, Interrupted }; constexpr std::string_view kColonCommands[] = { ":q", ":quit", ":exit", ":help", ":tables", ":scalars", ":functions", ":imports", ":schema", ":head", ":peek", ":describe", ":load", ":timing", - ":time", ":comments", ":doc", ":source", ":run", + ":time", ":comments", ":doc", ":source", ":run", ":explain", }; constexpr std::string_view kCompletionBuiltins[] = { @@ -2090,6 +2092,8 @@ void print_help() { " :peek Evaluate and compactly display an expression, with any\n" " order-sensitive claims it carries (time index, ordering,\n" " grouping)\n"); + ibex::formatting::print( + " :explain Show the read-only physical-plan capability for one expression\n"); ibex::formatting::print(" :describe
Schema + first rows\n"); ibex::formatting::print( " :doc Show docs/signature for a binding or built-in\n"); @@ -5309,6 +5313,72 @@ auto try_execute_whole_script(const parser::Program& program, runtime::ExternReg return true; } +/// Lower one REPL expression exactly as the evaluator would, then render the +/// physical planning capability without materializing a source or executing it. +void print_physical_explain(parser::Expr& expr, const runtime::TableRegistry& tables, + const LazyTableRegistry& lazy_tables, + const runtime::ScalarRegistry& scalars, const ColumnRegistry& columns, + const ModelRegistry& models, const FunctionRegistry& functions, + const CompileTimeListRegistry& compile_time_lists, + const ExternDeclRegistry& extern_decls, + const runtime::ExternRegistry& externs) { + parser::LowerContext context; + context.compile_time_lists = compile_time_lists; + for (const auto& [name, decl] : extern_decls) { + if (decl.return_type.kind == parser::Type::Kind::DataFrame || + decl.return_type.kind == parser::Type::Kind::TimeFrame) { + context.table_externs.insert(name); + context.table_extern_decls.insert_or_assign(name, &decl); + } + if (!decl.params.empty() && decl.params[0].type.kind == parser::Type::Kind::DataFrame) { + context.sink_externs.insert(name); + } + } + for (const auto& entry : scalars) { + context.lexical_names.insert(entry.first); + } + for (const auto& entry : columns) { + context.lexical_names.insert(entry.first); + } + for (const auto& entry : models) { + context.lexical_names.insert(entry.first); + } + for (const auto& entry : functions) { + context.lexical_names.insert(entry.first); + context.functions.insert_or_assign(entry.first, &entry.second); + } + for (const auto& entry : compile_time_lists) { + context.lexical_names.insert(entry.first); + } + for (const auto& entry : tables) { + context.lexical_names.insert(entry.first); + context.source_schemas.insert_or_assign(entry.first, table_schema_info(entry.second)); + } + for (const auto& entry : lazy_tables) { + context.lexical_names.insert(entry.first); + context.source_schemas.insert_or_assign(entry.first, + table_schema_info(entry.second->schema())); + } + + auto lowered = parser::lower_expr(expr, context); + if (!lowered.has_value()) { + ibex::formatting::print("error: {}\n", lowered.error().message); + return; + } + lowered.value() = + ir::push_filters_into_joins(std::move(lowered.value()), context.source_schemas); + lowered.value() = ir::push_semi_joins_down(std::move(lowered.value()), context.source_schemas); + lowered.value() = + ir::reduce_inner_joins_to_semi(std::move(lowered.value()), context.source_schemas); + const ir::OptimizationContext optimization_context; + lowered.value() = ir::optimize_plan(std::move(lowered.value()), optimization_context); + + const auto plan = runtime::physical::plan_physical(*lowered.value(), tables, &externs, + context.source_schemas); + ibex::formatting::print("Physical plan (capability; runtime fan-out may differ):\n{}", + runtime::physical::explain_physical(plan)); +} + } // namespace auto normalize_input(std::string_view input) -> std::string { @@ -5693,6 +5763,27 @@ void run(const ReplConfig& config, runtime::ExternRegistry& registry) { print_table(table.value(), count); continue; } + if (starts_with_command(line_view, ":explain")) { + auto source = trim(line_view.substr(std::string_view(":explain").size())); + if (source.empty()) { + ibex::formatting::print("usage: :explain \n"); + continue; + } + auto parsed = parser::parse(normalize_input(source)); + if (!parsed.has_value()) { + ibex::formatting::print("error: {}\n", parsed.error().format()); + continue; + } + if (parsed->statements.size() != 1 || + !std::holds_alternative(parsed->statements.front())) { + ibex::formatting::print("error: :explain expects a single expression\n"); + continue; + } + auto& expr = std::get(parsed->statements.front()).expr; + print_physical_explain(*expr, tables, lazy_tables, scalars, columns, models, functions, + compile_time_lists, extern_decls, registry); + continue; + } // Accept the obvious typo `:peak` as an alias for `:peek`. const bool is_peek = line_view.starts_with(":peek") && (line_view.size() == 5 || line_view[5] == ' ' || line_view[5] == '\t'); @@ -5839,7 +5930,8 @@ void run(const ReplConfig& config, runtime::ExternRegistry& registry) { ibex::formatting::print("error: unknown REPL command '{}'\n", cmd); ibex::formatting::print( "known: :help, :tables, :scalars, :functions, :imports, :schema, :head, " - ":peek, :describe, :doc, :source, :load, :timing, :time, :comments, :quit\n"); + ":peek, :explain, :describe, :doc, :source, :load, :timing, :time, :comments, " + ":quit\n"); continue; } diff --git a/src/runtime/CMakeLists.txt b/src/runtime/CMakeLists.txt index 91512d88..6c96155a 100644 --- a/src/runtime/CMakeLists.txt +++ b/src/runtime/CMakeLists.txt @@ -3,8 +3,10 @@ add_library(ibex_runtime STATIC aggregate.cpp + aggregate_chunked.cpp chunk_conversion.cpp chunked.cpp + join_chunked.cpp env.cpp execution_profile.cpp expr.cpp @@ -17,8 +19,10 @@ add_library(ibex_runtime STATIC lazy_table.cpp model.cpp ops.cpp + physical_executor.cpp physical_plan.cpp pipeline.cpp + pipeline_executor.cpp reshape.cpp rng.cpp runtime_internal.cpp diff --git a/src/runtime/PARALLELISM.md b/src/runtime/PARALLELISM.md index 3a953493..1e7302c7 100644 --- a/src/runtime/PARALLELISM.md +++ b/src/runtime/PARALLELISM.md @@ -107,7 +107,7 @@ project / rename / row-local update / their fused forms) run as **one independent task per morsel**, with an **order-preserving merge** that makes the output byte-identical to the serial chain regardless of completion order. -`MorselPipelineOperator` (`chunked.cpp`) is the executor. Key rules: +`MorselPipelineOperator` (`pipeline_executor.cpp`) is the executor. Key rules: 1. **Materialize the input subtree first, on the calling thread.** A deferred/lazy source decodes exactly once, serially, before any worker @@ -154,8 +154,9 @@ scatter** for anything variable-width. ## Who owns which decision -The honest answer is **it depends on the operator category**, and only one -category has a single clean owner today. +The answer depends on the operator category. Map chains and the promoted +Distinct, Join, and Aggregate policies have a clean plan/executor split; +several remaining breaker internals do not. ### The stable parts @@ -171,52 +172,48 @@ category has a single clean owner today. |---|---|---|---| | **Map chains** (Filter/Project/Rename/row-local Update, fused) | **The physical planner owns it end to end.** `plan.mode` (`Serial`/`MorselParallel`), `parallel_begin`/`parallel_end` (which steps run over morsels), and per-step `MapStep` (capability + kernel factory + column signature). | **Yes, fully.** | Yes. | | **Join** | The plan owns the **structure** — `JoinPlan` carries build side + runtime-resolved orientation (`49188c71`) — and **both** fan-out phases: `build_partitions` reads `par_.build`, `probe_parallel_workers` reads `par_.probe` (slices 4–5). What stays in the operator is the kill switch / nesting / per-chunk floor. Output assembly is inside `ChunkedInnerJoinOperator`. | **Yes** (structure + both phases, both authoritative). | Structure + both phases. | -| **Aggregate** | The plan owns `AggregatePlan` (which construction path). **Discovery, per-partition slots, the owned-aggregate hot table, and the finalize merge are inside `ChunkedAggregateOperator`.** | Partial. | Which path, not the phases. | -| **Distinct / Order / TopK / Head / Tail** | **Nothing in the plan.** `execution_capability(Distinct)` returns `ParallelBarrier`, but that value is **never read to make a decision** — it names what a future executor *could* do. `build_physical_distinct` just constructs `ChunkedDistinctOperator`, which owns the entire decision internally: the `can_fan_out()` / `on_worker_pool_thread()` guard, a private `kMinRows` (hardcoded, and twice — accumulate and finalize), the partition count from `compute_budget()`, the two-pass "one worker per partition scans the whole chunk" model. | **No.** | **No.** | -| **Layer C fan-out inside any operator** (group discovery, sort gather, decode, semi/anti predicate) | Always the operator's, each with its own private row threshold and its own `min(budget, pool, cap)` worker count. | No. | No. | - -**So: there is no single owner of "distinct parallelism" — and the same is true -of every barrier operator's parallelism.** For map chains the physical plan is -that owner; for breakers it owns construction (and for join/aggregate, -structure), and the parallel-execution decisions live inside the operator, -unrepresented, un-inspectable, and tunable only by editing the operator. Closing -that gap — decomposing the breakers into planned phases — is -`kernel-pipeline-execution-plan.md` Phase 4, which is why that plan distinguishes -"construction ownership done" (backlog 116→6) from "decomposition not started". - -### The one rule that already holds everywhere - -**The plan says whether parallel execution is *permitted*; the operator says -whether it is *desirable* here** (`exec.can_fan_out()`, row/cell floors, morsel -count, `on_worker_pool_thread()`). `plan.mode == MorselParallel` is a -capability, and a serial execution of that plan (`can_fan_out()` false) must -still be correct — a q19 crash under `IBEX_CORES=1` came from an executor that -checked only `plan.mode`. For breakers, both halves currently live in the -operator. +| **Aggregate** | The plan owns the adaptive strategy, positional `AggregateColumnMapping`, and four typed hash-fallback nodes (discovery → accumulation → final ordering → emission). Each node carries and authoritatively supplies its own fan-out policy. A serial coordinator invokes every node: discovery transfers one bounded chunk of group IDs/column bindings to accumulation, or marks a one-pass kernel explicitly fused; final ordering and emission are separate. | **Yes, structural and authoritative.** | Strategy + typed chain + all four node policies. | +| **Distinct** | The plan owns the `dedup` policy (floor, ceiling, packed-key strategy, optional estimate); the builder resolves it and the operator reads it. Nesting, the first concrete chunk's row count, and the derived partition count remain runtime decisions. | **Yes, authoritative.** | Yes. | +| **Order** | The plan describes one `sort` phase. The actual radix-sort/gather fan-out reads shared `ExecutionContext` knobs in `sort.cpp`, so this phase is descriptive rather than authoritative. | **Yes, descriptive.** | Yes. | +| **TopK / Head / Tail / FilterHead / FilterTail** | Plan-built serial breaker operators. They have no current fan-out point; TopK deliberately uses a bounded streaming heap rather than a full sort. | **Yes; no parallel policy.** | Serial-by-design reason. | +| **Remaining Layer C fan-out inside operators** (sort gather, decode, semi/anti predicate, and data-dependent aggregate specialization gates) | The operator owns decisions not yet promoted. Migrated Distinct, Join, and Aggregate paths instead read their resolved plan policy and retain only runtime/data-dependent admission checks. | Mixed. | Only promoted phases. | + +**There is not yet one owner for every breaker's parallelism.** The physical +plan owns map chains and the promoted Distinct, Join, and Aggregate policies; +other breaker internals remain unrepresented and tunable only at their use +sites. Aggregate's former coarse `partition` / `finalize` records are gone; +its remaining coordinator is deliberately serial while the work inside each +typed node may fan out under that node's policy. + +### The split enforced by migrated parallel paths + +**The plan says whether parallel execution is *permitted*; the operator handles +facts available only at runtime** (actual rows/cells, morsel count, +`on_worker_pool_thread()`, and data-dependent specialization gates). +`plan.mode == MorselParallel` is a capability, and a serial execution of that +plan (`can_fan_out()` false) must still be correct — a q19 crash under +`IBEX_CORES=1` came from an executor that checked only `plan.mode`. Breakers not +yet promoted still combine these halves at their use sites. ## Worked example: `t[distinct { g, v }]` 1. **Logical IR** — a `Distinct` node over a `Scan`. The optimizer decides column demand (`g`, `v`), nothing about execution. -2. **`plan_physical`** — the root is `Distinct`, not a map chain, so - `plan.migrated == false`, `reason == NotMapChain`. `plan.mode` is irrelevant - (it's for map chains). No `DistinctPlan` field exists. `explain physical` - prints `MaterializedCall(Distinct)` — or, since `49ca33c1`, records it as a - plan-built breaker — and says nothing about how it will run. -3. **`build_physical_distinct`** — constructs `ChunkedDistinctOperator(child, - exec)`. Passes the `ExecutionContext` in; makes no parallelism decision. -4. **`ChunkedDistinctOperator`, first chunk** — decides *everything*: if - `!exec.can_fan_out() || on_worker_pool_thread() || rows < 32768` it stays - serial and pins `dedup_part_count_ = 1` for all later chunks; otherwise it - derives `part_count` from `compute_budget()`, hash-partitions by packed key, - and runs one worker per partition (each scanning the whole chunk, skipping - rows not in its partition — the proven Pass-2 model). Determinism device: - workers record a keep-flag at a row, never a position, so the output is - rebuilt by scanning flags in row order. - -The only externally visible knob is `IBEX_CORES` (via `can_fan_out()` and -`compute_budget()`). The `32768` floor and the partition strategy are editable -only in `chunked.cpp`. +2. **`plan_physical`** — records a migrated `Breaker(Distinct)` with one + `dedup` phase: packed-key strategy, 32768-row floor, worker ceiling, and any + available row estimate. `explain physical` renders that unresolved policy. +3. **`build_physical_distinct`** — resolves the policy against the + `ExecutionContext` and worker pool, then passes it to + `ChunkedDistinctOperator`. +4. **`ChunkedDistinctOperator`, first chunk** — reads the resolved permission + and cap, applies the facts only it knows (nesting and actual rows), and pins + a derived partition count for later chunks. Each worker scans the chunk for + its partition. Workers record keep-flags by row, so output is rebuilt in + input order rather than completion order. + +The externally visible compute knob is `IBEX_CORES`; the 32768-row floor and +packed-key strategy are named once by `distinct_dedup_parallelism` and carried +by the physical plan. ## The determinism contract @@ -237,11 +234,10 @@ The devices: The only legitimate exceptions: PDS-H q01/q09/q15 differ by ≤1 ulp from parallel float reduction order (itself thread-count-independent), enumerated in -`beat-polars-plan.md` §5. Anything else that differs is a bug. **Known standing -violation:** the two-Int64-key owned aggregate (`try_owned_pair`) and the serial -path re-associate differently and disagree bit-for-bit at ≥65536 rows, with no -test covering it — `kernel-pipeline-execution-plan.md` "The determinism -constraint is already broken". +`beat-polars-plan.md` §5. Anything else that differs is a bug. The former +two-Int64-key owned-aggregate divergence no longer reproduces; the +"two-key grouped aggregate is deterministic across thread counts" regression +test now guards the serial and parallel paths. ## Configuration surface @@ -265,14 +261,14 @@ disagree. ## Where the model is still muddy -**The structural one:** breaker parallelism has no owner above the operator (see -"Who owns which decision"). A breaker's fan-out decision, its partition count, -and its worker cap are private to `chunked.cpp`, invisible to `explain -physical`, and un-A/B-able except through `IBEX_CORES`. Every other item below is -a symptom of that — the private thresholds and worker caps exist because there -is no plan-level place to put them. Fixing the altitude (decomposing breakers -into planned phases, `kernel-pipeline` Phase 4) is what makes the rest -tractable; fixing the symptoms first just moves nine constants into one header. +**The structural owner now exists for migrated breakers.** Distinct, streaming +Join, and streaming Aggregate take their fan-out policies from explicit +physical nodes and expose those decisions through `explain physical`. +Aggregate's four phases, streaming inner join, migrated-plan dispatch, and the +generic map/morsel executor now live outside `chunked.cpp`. Remaining +operator-private decisions belong to breaker families that have not completed +that migration; do not generalize their local thresholds into a second policy +system. **The symptoms** (`plans/parallelism-overview.md` Part 2 is the live catalogue): type-exclusion rules with no shared "is this type parallel-capable @@ -349,9 +345,9 @@ struct BreakerParallelism { RowEstimate estimate{}; }; -// One breaker = one or more named phases, each with its own fan-out point. -// Distinct/Order/TopK have one; a decomposed Aggregate has three -// (discovery / accumulate / finalize); a Join has two (hash-build / probe). +// Untyped breakers retain named phases. Join and Aggregate instead carry the +// same descriptor directly on their typed HashBuild/HashProbe and +// Discovery/Accumulation/FinalOrdering/Emission nodes. struct BreakerPhase { std::string_view name; BreakerParallelism parallelism; @@ -440,8 +436,9 @@ Every slice: once. - **Not removing the runtime checks.** `on_worker_pool_thread()` and the first-chunk floor check are the operator's, permanently. -- **Not the `chunked.cpp` split.** That is Phase 5, and it is deliberately - *after* this — the plan says decomposition makes the split easier. +- **Not ownership of the remaining `chunked.cpp` split.** This contract enabled + Phase 5; Aggregate and streaming inner join have now moved, while + planner/executor extraction is tracked by the kernel-pipeline plan. - **Not a row-count estimator project.** The estimate is opportunistic (footer stats, exact child counts). `partition_count = 0 / derive` is the honest default and preserves today's behavior exactly. @@ -495,10 +492,8 @@ Every slice: onto the plan, byte-identical throughout. The `probe_parallel_workers` `on_worker_pool_thread()` veto was measured to fire 0/52 on PDS-H, so folding it changed nothing. -4. **Aggregate** — `AggregatePlan` gains `partition` + `finalize` phases (the - two fan-out points `ChunkedAggregateOperator` has today; discovery and - accumulate are one `pool.submit`, so they are one phase until that region is - actually decomposed in Phase 5). **Determinism blocker cleared (2026-08-27):** +4. **Aggregate** — four typed structural nodes now own the hash fallback's + scheduling policy. **Determinism blocker cleared (2026-08-27):** the `try_owned` vs serial re-association divergence recorded below does not reproduce on the current tree — the serial probe path, the owned path, and a strict-row-order reference all agree bit-for-bit at every thread count @@ -508,30 +503,22 @@ Every slice: `d5928ee2`) reconciled it. Removing `try_owned`'s schedule gate outright was also tried and reverted — correctness stayed byte-identical but 1-core q20/q18 regressed +25%/+40%. The guard test now exists: `tests/test_interpreter.cpp` - "two-key grouped aggregate is deterministic across thread counts". Slice 1 - (observability) LANDED: `aggregate_{partition,finalize}_parallelism`, - `plan_physical` fills the phases, `explain physical` prints them, - `ChunkedAggregateOperator::check_agg_plan` aborts on planner/operator - disagreement — byte-identical, full suite + q01/q10/q13/q18/q20/q21 at 1c/8c. - Slice 2 (partition authority) LANDED: `try_owned` and `try_discover_ - partitioned` read `par_.partition.{decline,worker_cap}` for fan-out - permission and the worker cap; the open-coded `!can_fan_out()` + - `min(budget, pool, 64)` are gone from both. The floors stay in the operator - — the radix `kDefaultPartitionMinRows` beside its constant, `try_owned`'s - lower `kPairOwnedMinRows` as the operator-resolved "owned specialization - worth it" gate (like the join's build orientation). Byte-identical vs base - on q01/q10/q13/q18/q20/q21, full suite. - - Slice 3 (finalize authority + the async-hot partition gate slice 2 missed) - LANDED: `finalize_owned`'s co-ranking merge, the ordered-run merge, the - first-occurrence seed pass, and the async-hot cold build read - `par_.finalize.{decline,worker_cap}` for their ceiling and permission; the - data-derived cap terms (`part_count`, `total/4096`, `run_count/8192`) and the - three strategy floors (`1U<<17`, `1U<<16`, `parallel_min_rows`) stay in the - operator. `try_async_hot_int_sum` (the q18 path — a fourth `partition` - strategy slice 2 did not touch) reads `par_.partition` too now. New - `ParallelPipelineStats` counters `parallel_aggregate_partitions` / - `parallel_aggregate_finalizes` so a silently-stopped gate is a red test. - Byte-identical vs the slice-1 base on q01/q10/q13/q18/q20/q21, full suite - 1814/1814. **The Aggregate step of "parallelism as a plan decision" is - complete.** + "two-key grouped aggregate is deterministic across thread counts". The + observability and authority migration removed the open-coded + `!can_fan_out()` / `min(budget, pool, 64)` decisions while retaining + data-derived caps (`part_count`, `total/4096`, `run_count/8192`) and + strategy-specific admission floors next to their kernels. Existing + `ParallelPipelineStats` partition/finalize counters remain compatibility + telemetry; structural execution-profile rows are the per-node proof. The + executor mutation seam verifies mapped positions, typed edges, and worker + ceilings are consumed rather than reconstructed locally. + + Slice 4 (structural-node authority) LANDED 2026-08-29: the coarse records + are removed. Discovery owns radix, owned, and async-hot group discovery; + Accumulation owns deterministic private-state morsels (including fused + categorical/global kernels) and slot initialization; FinalOrdering owns + ordered merges and first-occurrence seeding; Emission owns independent + output-column tasks. `explain physical` renders + all four policies on their typed nodes. Execution-profile mutation tests + prove a one-worker node ceiling suppresses that node's pool work without + reconstructing defaults, while preserving byte-identical output. diff --git a/src/runtime/aggregate_chunked.cpp b/src/runtime/aggregate_chunked.cpp new file mode 100644 index 00000000..79a75ff8 --- /dev/null +++ b/src/runtime/aggregate_chunked.cpp @@ -0,0 +1,5349 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +// Adaptive sorted/hash aggregate execution. The complete aggregate family is +// kept in this translation unit so extraction from the pipeline builder does +// not split hot templates or state across compilation boundaries. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "physical_plan.hpp" + +#if defined(__AVX2__) || defined(__BMI2__) +#include +#endif + +#include "aggregate_chunked_internal.hpp" +#include "chunk_conversion_internal.hpp" +#include "execution_profile_internal.hpp" +#include "interpreter_internal.hpp" +#include "packed_key_encoder_internal.hpp" +#include "runtime_internal.hpp" + +namespace ibex::runtime { + +namespace { + +// Whether a streamed aggregate slot has enough observations to be non-null. +// Mirrors the materializing aggregate's `agg_result_is_valid`. +auto chunked_agg_valid(ir::AggFunc func, const AggSlotCore& slot) -> bool { + switch (func) { + case ir::AggFunc::Mean: + return slot.count > 0; + case ir::AggFunc::Sum: + case ir::AggFunc::Min: + case ir::AggFunc::Max: + case ir::AggFunc::First: + case ir::AggFunc::Last: + return slot.present(); + case ir::AggFunc::Stddev: + return slot.count >= 2; + case ir::AggFunc::Skew: + return slot.count >= 3; + case ir::AggFunc::Kurtosis: + return slot.count >= 4; + default: // Count + return true; + } +} + +// Whether a streamed aggregate carries a validity bitmap at all (Count never +// produces nulls; the value-bearing aggs may). +auto chunked_agg_tracks_validity(ir::AggFunc func) -> bool { + switch (func) { + case ir::AggFunc::Sum: + case ir::AggFunc::Mean: + case ir::AggFunc::Min: + case ir::AggFunc::Max: + case ir::AggFunc::First: + case ir::AggFunc::Last: + case ir::AggFunc::Stddev: + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + return true; + default: + return false; + } +} + +/// Streaming hash aggregate. Maintains a `robin_hood` group index and +/// per-group `AggState` across chunks: each incoming chunk updates the +/// state per row, the chunk is released, and the final result is +/// emitted as a single output chunk on EOF. +/// +/// Eligibility is gated at `build_operator` time to the common subset +/// that streams cleanly: `Count`, `Sum`, `Min`, `Max`, `Mean` on +/// numeric (int/double) inputs. Nullable agg inputs are handled — null +/// rows skip the update, and an all-null group emits a null result. +/// Nullable group-by columns are not supported yet; they fall back to +/// `aggregate_table` via `interpret_node`. Complex aggs (Median, etc.) +/// and string aggs also fall back. +/// +/// The first chunk's group-by column types are snapshotted (including +/// the Categorical dictionary pointer when applicable) and reused when +/// building output; the chunked csv source shares dictionaries across +/// chunks, matching MaterializeOperator's existing assumption. +/// A growable array of trivially-copyable slots that grows through `realloc`. +/// +/// `std::vector` cannot use `realloc`: it must allocate, copy, and free, and on +/// this array that copy IS the cost. A group-by discovers its groups a chunk at +/// a time, so the slot array is resized once per chunk and never shrinks; by +/// the last chunk the copies dominate. Measured on q18 (3M groups over 6 +/// chunks): `size_group_arrays` cost 79ms, and pre-reserving the final size -- +/// which a real query cannot do, since the group count is what it is about to +/// find out -- removed 49ms of it. That removed cost is all copying. +/// +/// At these sizes the block is served by `mmap`, and `realloc` extends it with +/// `mremap`: page-table work, no bytes moved. The elements it must still touch +/// are only the NEW ones, which is the irreducible part. +/// +/// Deliberately minimal: no shrink, no insert, no iterators. It is a slot array +/// indexed by group id, and every use it has is `resize` / `data` / `[]`. +/// +/// The `realloc`/`free` calls are the whole reason this class exists (in-place +/// `mremap` growth, no copy) -- it is itself the RAII wrapper the check wants. +// NOLINTBEGIN(cppcoreguidelines-no-malloc) +template +class SlotArray { + public: + static_assert(std::is_trivially_copyable_v); + static_assert(std::is_trivially_destructible_v); + + SlotArray() = default; + SlotArray(const SlotArray&) = delete; + auto operator=(const SlotArray&) -> SlotArray& = delete; + SlotArray(SlotArray&& other) noexcept + : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { + other.data_ = nullptr; + other.size_ = 0; + other.capacity_ = 0; + } + auto operator=(SlotArray&& other) noexcept -> SlotArray& { + if (this != &other) { + std::free(data_); + data_ = other.data_; + size_ = other.size_; + capacity_ = other.capacity_; + other.data_ = nullptr; + other.size_ = 0; + other.capacity_ = 0; + } + return *this; + } + ~SlotArray() { std::free(data_); } + + [[nodiscard]] auto size() const noexcept -> std::size_t { return size_; } + [[nodiscard]] auto data() noexcept -> T* { return data_; } + [[nodiscard]] auto data() const noexcept -> const T* { return data_; } + auto operator[](std::size_t i) noexcept -> T& { return data_[i]; } + auto operator[](std::size_t i) const noexcept -> const T& { return data_[i]; } + + /// Grow to `n` WITHOUT initializing the new tail, which is returned for the + /// caller to fill. Never shrinks the allocation: a group-by only ever adds + /// groups. + /// + /// Split out of `resize` because on a large array the fill is the expensive + /// half and it is not serial by nature. `mremap` hands back pages the kernel + /// has yet to materialize, so writing them is 72MB of first-touch page + /// faults on q18's 3M slots — 44ms in a cold process, several times what the + /// bytes alone cost, and page faults are what scales with threads. A caller + /// holding a worker pool fans this out; one without it calls `resize` and + /// pays the serial fill. + /// + /// How much this is worth depends on whether the pages are fresh, so read + /// the two numbers separately. Cold (`ibex query.ibex`, the path a script + /// takes) the fan-out takes q18's fill 44ms -> 34ms and the whole query + /// -4.3%. Warm — the PDS-H harness, which reuses one process, so the + /// allocator hands back pages already faulted — the fill is plain bandwidth + /// and the suite geomean does not move. + [[nodiscard]] auto grow_uninitialized(std::size_t n) -> std::span { + if (n <= size_) { + size_ = n; + return {}; + } + if (n > capacity_) { + // Geometric, so a per-chunk resize does not call realloc once per + // chunk on a stream of many small chunks. + const std::size_t want = std::max(n, capacity_ + (capacity_ / 2)); + auto* grown = static_cast(std::realloc(data_, want * sizeof(T))); + if (grown == nullptr) { + throw std::bad_alloc(); + } + data_ = grown; + capacity_ = want; + } + const std::size_t old = size_; + size_ = n; + return {data_ + old, n - old}; + } + + /// Value-initialize `tail`, a range `grow_uninitialized` just handed back. + /// + /// One `memset` when `T`'s value-initialized form is all-zero bytes, which + /// every slot type here is (`AggSlotCore`'s two enums both start at 0). The + /// per-element copy this replaces cost q18's fill 12ms of its 55: three + /// million 24-byte `memcpy`s the compiler will not fuse, because it cannot + /// see that the prototype is zeros. + /// + /// Padding is why the test is a run-time `memcmp` rather than a + /// `static_assert`: value-initialization zeroes `T`'s padding too, so the + /// comparison is well defined here, but no constant expression can state + /// that for a type with padding. The loop keeps the class honest for a + /// future slot type whose default is not all zeros. + static void fill_default(std::span tail) noexcept { + if (tail.empty()) { + return; + } + const T prototype{}; + alignas(T) std::array zero{}; + // prototype{} zeroes padding too (see the class comment), so this is well defined. + // NOLINTNEXTLINE(cert-exp42-c,cert-flp37-c,bugprone-suspicious-memory-comparison) + if (std::memcmp(&prototype, zero.data(), sizeof(T)) == 0) { + // Through `void*`: `T` has default member initializers, so it is not + // trivially default-constructible and -Wclass-memaccess objects to + // memset-ing it directly. The memcmp above is what licenses this. + std::memset(static_cast(tail.data()), 0, tail.size() * sizeof(T)); + return; + } + for (auto& slot : tail) { + std::memcpy(&slot, &prototype, sizeof(T)); + } + } + + /// Grow to `n`, value-initializing the new tail on the calling thread. + void resize(std::size_t n) { fill_default(grow_uninitialized(n)); } + + private: + T* data_ = nullptr; + std::size_t size_ = 0; + std::size_t capacity_ = 0; +}; +// NOLINTEND(cppcoreguidelines-no-malloc) + +auto bind_aggregate_columns(std::optional& columns, bool& bound, + const std::vector& group_by, + const std::vector& aggregations, const Chunk& chunk) + -> std::optional { + if (bound) { + return std::nullopt; + } + std::vector names; + names.reserve(chunk.columns.size()); + for (const ColumnEntry& column : chunk.columns) { + names.push_back(column.name); + } + const bool concrete_layout_matches_plan = columns.has_value() && + columns->input_names.size() == names.size() && + std::ranges::equal(columns->input_names, names); + if (!concrete_layout_matches_plan) { + // Logical schema inference may know every source column while the + // physical child emits a narrower layout. A pushed-down filter, for + // example, can consume its predicate-only column inside a lazy scan and + // omit it from the chunks delivered to this breaker. Bind that actual + // boundary once; all row loops remain positional. + auto resolved = physical::resolve_aggregate_columns(group_by, aggregations, names); + if (!resolved.has_value()) { + return std::move(resolved.error()); + } + columns = std::move(*resolved); + } + if (columns->group_by.size() != group_by.size() || + columns->aggregate_inputs.size() != aggregations.size()) { + return "aggregate column mapping does not match aggregate shape"; + } + for (std::size_t i = 0; i < columns->group_by.size(); ++i) { + const std::size_t index = columns->group_by[i]; + if (index >= chunk.columns.size() || chunk.columns[index].name != group_by[i].name) { + return "aggregate group-by column mapping does not match concrete input"; + } + } + for (std::size_t i = 0; i < columns->aggregate_inputs.size(); ++i) { + const auto index = columns->aggregate_inputs[i]; + if (aggregations[i].func == ir::AggFunc::Count) { + if (index.has_value()) { + return "count aggregate unexpectedly has an input column mapping"; + } + continue; + } + if (!index.has_value() || *index >= chunk.columns.size() || + chunk.columns[*index].name != aggregations[i].column.name) { + std::string detail = + "aggregate input column mapping does not match concrete input: expected '" + + aggregations[i].column.name + "'"; + if (index.has_value()) { + detail += " at position " + std::to_string(*index); + if (*index < chunk.columns.size()) { + detail += ", found '" + chunk.columns[*index].name + "'"; + } else { + detail += ", but the input has only " + std::to_string(chunk.columns.size()) + + " columns"; + } + } + return detail; + } + } + bound = true; + return std::nullopt; +} + +class HashAggregateState final { + public: + /// `Cat` carries a Categorical's *code*, which the pair path may treat as + /// an integer for the same reason `process_rows_cat` may index an array + /// with it: within one operator a dictionary only ever grows and never + /// reorders, so a code identifies the same value in every chunk. + enum class IntKeyKind : std::uint8_t { Int64, Date, Ts, Cat }; + + HashAggregateState(OperatorPtr child, const std::vector* group_by, + const std::vector* aggregations, const ExecutionContext& exec, + physical::AggregateParallelism par = {}, + std::optional columns = std::nullopt) + : child_(std::move(child)), + group_by_(group_by), + aggregations_(aggregations), + exec_(&exec), + columns_(std::move(columns)), + par_(par), + discovery_profile_(exec.execution_profile == nullptr + ? nullptr + : exec.execution_profile->stage("Aggregate.Discovery")), + accumulation_profile_(exec.execution_profile == nullptr + ? nullptr + : exec.execution_profile->stage("Aggregate.Accumulation")), + final_ordering_profile_(exec.execution_profile == nullptr + ? nullptr + : exec.execution_profile->stage("Aggregate.FinalOrdering")), + emission_profile_(exec.execution_profile == nullptr + ? nullptr + : exec.execution_profile->stage("Aggregate.Emission")) {} + + /// Pull and run the structural Discovery node for one chunk. The chunk is + /// retained until `accumulate_discovery` consumes its transfer, so the + /// ColumnEntry pointers in that value remain valid without copying data. + auto next_discovery() -> std::expected { + if (input_consumed_) { + return false; + } + if (active_chunk_.has_value()) { + return std::unexpected( + "physical aggregate: Discovery advanced before Accumulation consumed its input"); + } + auto chunk_res = child_->next(); + if (!chunk_res.has_value()) { + return std::unexpected(std::move(chunk_res.error())); + } + if (!chunk_res.value().has_value()) { + input_consumed_ = true; + return false; + } + active_chunk_ = std::move(*chunk_res.value()); + const ExecutionProfileScope scope(discovery_profile_, ProfilePhase::Next); + if (auto err = discover_chunk(*active_chunk_)) { + return std::unexpected(*err); + } + return true; + } + + /// Consume the current Discovery output at the structural Accumulation + /// node, then release the input chunk before the source advances. + auto accumulate_discovery() -> std::expected { + if (!active_chunk_.has_value()) { + return std::unexpected("physical aggregate: Accumulation has no discovered chunk"); + } + const ExecutionProfileScope scope(accumulation_profile_, ProfilePhase::Next); + if (auto err = accumulate_discovered_chunk()) { + return std::unexpected(*err); + } + active_chunk_.reset(); + return {}; + } + + /// Structural FinalOrdering entry. Owned-partition strategies transfer + /// their local group state into deterministic first-occurrence order here; + /// already-global strategies have no deferred work at this boundary. + auto finalize_ordering() -> std::optional { + const ExecutionProfileScope scope(final_ordering_profile_, ProfilePhase::Next); + if (ordering_finalized_) { + return std::nullopt; + } + if (owned_mode_) { + finalize_owned_active(); + if (owned_async_error_.has_value()) { + return owned_async_error_; + } + } + ordering_finalized_ = true; + return std::nullopt; + } + + /// Structural Emission entry. It consumes only finalized, globally ordered + /// group state and constructs the result columns. + auto emit_output() -> std::expected, std::string> { + const ExecutionProfileScope scope(emission_profile_, ProfilePhase::Next); + if (!input_consumed_) { + return std::unexpected("physical aggregate: Emission ran before input consumption"); + } + if (active_chunk_.has_value()) { + return std::unexpected("physical aggregate: Emission ran with unconsumed discovery"); + } + if (!ordering_finalized_) { + return std::unexpected("physical aggregate: Emission ran before FinalOrdering"); + } + if (emitted_) { + return std::optional{}; + } + emitted_ = true; + return build_output_chunk(); + } + + private: + /// Telemetry for the breaker-parallelism slice (src/runtime/PARALLELISM.md). + /// The plan owns each phase's worker cap and fan-out permission now; the + /// fan-out output is byte-identical to serial, so without a counter a gate + /// that silently stopped matching would lose the parallelism with every + /// test green. Counted once per fan-out commit (per chunk for `partition`, + /// once for `finalize`, which runs once). + void note_partition_fanout() const { + if (exec_ != nullptr && exec_->parallel_stats != nullptr) { + exec_->parallel_stats->parallel_aggregate_partitions.fetch_add( + 1, std::memory_order_relaxed); + } + } + void note_finalize_fanout() const { + if (exec_ != nullptr && exec_->parallel_stats != nullptr) { + exec_->parallel_stats->parallel_aggregate_finalizes.fetch_add( + 1, std::memory_order_relaxed); + } + } + + enum class DiscoveryTransferKind : std::uint8_t { + None, + NeedsAccumulation, + FusedAccumulation, + }; + + /// Per-chunk ownership transfer from Discovery to Accumulation. Column + /// pointers remain valid because `consume_input` keeps the owning Chunk + /// alive until `accumulate_discovered_chunk` consumes this value. The gid + /// buffer is operator-owned and cannot be reused until the transfer resets. + struct AggregateDiscoveryTransfer { + DiscoveryTransferKind kind = DiscoveryTransferKind::None; + std::vector aggregate_entries; + std::vector skip_fields; + std::size_t rows = 0; + }; + + void publish_discovered(const std::vector& aggregate_entries, + std::size_t rows, + const std::vector* skip_fields = nullptr) { + discovery_transfer_.kind = DiscoveryTransferKind::NeedsAccumulation; + discovery_transfer_.aggregate_entries = aggregate_entries; + discovery_transfer_.rows = rows; + discovery_transfer_.skip_fields = + skip_fields == nullptr ? std::vector{} : *skip_fields; + } + + void publish_fused_accumulation() { + discovery_transfer_ = {}; + discovery_transfer_.kind = DiscoveryTransferKind::FusedAccumulation; + } + + auto accumulate_discovered_chunk() -> std::optional { + if (discovery_transfer_.kind == DiscoveryTransferKind::None) { + return "physical aggregate: Discovery produced no accumulation transfer"; + } + if (discovery_transfer_.kind == DiscoveryTransferKind::FusedAccumulation) { + discovery_transfer_ = {}; + return std::nullopt; + } + if (gids_buf_.size() < discovery_transfer_.rows) { + return "physical aggregate: Discovery produced a short group-id buffer"; + } + const auto* skip = + discovery_transfer_.skip_fields.empty() ? nullptr : &discovery_transfer_.skip_fields; + accumulate_gids(gids_buf_.data(), discovery_transfer_.aggregate_entries, + discovery_transfer_.rows, skip); + discovery_transfer_ = {}; + return std::nullopt; + } + + auto discover_chunk(const Chunk& chunk) -> std::optional { + discovery_transfer_ = {}; + if (std::getenv("IBEX_AGG_PARTITION_DEBUG") != nullptr) { + ibex::formatting::print(stderr, "[agg_process_chunk] rows={} group_by_size={}\n", + chunk.rows(), group_by_->size()); + } + // Counted here, once per chunk, because the partition gate below asks + // how much input this OPERATOR has — a question the per-call row count + // stopped answering the moment sources began arriving in pieces. + rows_offered_ += chunk.rows(); + if (auto err = bind_aggregate_columns(columns_, columns_bound_, *group_by_, *aggregations_, + chunk)) { + return err; + } + // `bind_aggregate_columns` only returns nullopt once `columns_` is bound. + if (!columns_.has_value()) { + return "HashAggregateState: column mapping not bound"; + } + const physical::AggregateColumnMapping& cols = *columns_; + std::vector group_entries; + group_entries.reserve(group_by_->size()); + for (const std::size_t index : cols.group_by) { + group_entries.push_back(&chunk.columns[index]); + } + + std::vector agg_entries(aggregations_->size(), nullptr); + for (std::size_t i = 0; i < aggregations_->size(); ++i) { + const auto& agg = (*aggregations_)[i]; + const std::optional& input_idx = cols.aggregate_inputs[i]; + if (agg.func == ir::AggFunc::Count || !input_idx.has_value()) { + continue; + } + const ColumnEntry* entry = &chunk.columns[*input_idx]; + const ExprType kind = expr_type_for_column(*entry->column); + const bool first_or_last = + agg.func == ir::AggFunc::First || agg.func == ir::AggFunc::Last; + // First/Last also accept String (which covers Column and + // Column — expr_type_for_column collapses both to + // String); every other function stays numeric-only. + const bool supported = kind == ExprType::Int || kind == ExprType::Double || + (first_or_last && kind == ExprType::String); + if (!supported) { + return "HashAggregateState: non-numeric aggregation not supported"; + } + agg_entries[i] = entry; + } + + if (!initialized_) { + n_aggs_ = aggregations_->size(); + plan_.reserve(n_aggs_); + for (std::size_t i = 0; i < n_aggs_; ++i) { + SlotPlan p; + p.func = (*aggregations_)[i].func; + if (p.func == ir::AggFunc::Count) { + p.kind = ExprType::Int; + } else { + p.kind = expr_type_for_column(*agg_entries[i]->column); + p.categorical = + std::holds_alternative>(*agg_entries[i]->column); + } + plan_.push_back(p); + } + // Lay the scratch out once the plan is known. Skew/Kurtosis share + // one online recurrence that updates both higher moments, so each + // asks for the pair. + scratch_offset_.assign(n_aggs_, 0); + scratch_stride_ = 0; + for (std::size_t i = 0; i < n_aggs_; ++i) { + // Scratch layout is [m2, m3, m4]. Stddev needs only the + // first; the higher moments imply it, since their recurrence + // reads m2 on every update. + if (plan_[i].func == ir::AggFunc::Stddev) { + plan_[i].scratch_doubles = 1; + } else if (plan_[i].func == ir::AggFunc::Skew || + plan_[i].func == ir::AggFunc::Kurtosis) { + plan_[i].scratch_doubles = 3; + } + scratch_offset_[i] = static_cast(scratch_stride_); + scratch_stride_ += plan_[i].scratch_doubles; + } + group_templates_.reserve(group_entries.size()); + bool all_cat = true; + for (const auto* e : group_entries) { + group_templates_.push_back(make_empty_like(*e->column)); + if (!std::holds_alternative>(*e->column) || + e->validity.has_value()) { + all_cat = false; + } + } + cat_fast_path_ = all_cat && !group_entries.empty(); + // Single-string-key fast path: avoids the generic `Key`/ScalarValue + // variant path used by `process_rows_generic`. High-cardinality + // `sum by user_id` (~100K distinct strings in 2M rows) was spending + // most of its time constructing per-row ScalarValue variants and + // hashing them; the string path uses a string_view map keyed against + // an owned char/offset dictionary instead. + str_fast_path_ = + group_entries.size() == 1 && + std::holds_alternative>(*group_entries[0]->column) && + !group_entries[0]->validity.has_value(); + // Single fixed-width-integer key: a direct value map, no owned Key. + const auto int_kind_of = [](const ColumnValue& col) -> std::optional { + if (std::holds_alternative>(col)) { + return IntKeyKind::Int64; + } + if (std::holds_alternative>(col)) { + return IntKeyKind::Date; + } + if (std::holds_alternative>(col)) { + return IntKeyKind::Ts; + } + return std::nullopt; + }; + if (group_entries.size() == 1 && !group_entries[0]->validity.has_value()) { + if (auto kind = int_kind_of(*group_entries[0]->column)) { + int_fast_path_ = true; + int_key_kind_ = *kind; + } + } else if (!cat_fast_path_ && group_entries.size() == 2 && + !group_entries[0]->validity.has_value() && + !group_entries[1]->validity.has_value()) { + // A Categorical joins the pair path as its code. `cat_fast_path_` + // already owns the all-Categorical case and is dispatched first, + // so this is reached only by a *mixed* pair — `by { symbol, day }` + // over a Categorical and a Date, which otherwise fell to the + // generic path and hashed the symbol as text once per row. + const auto pair_kind_of = [&](const ColumnValue& col) -> std::optional { + if (std::holds_alternative>(col)) { + return IntKeyKind::Cat; + } + return int_kind_of(col); + }; + auto ka = pair_kind_of(*group_entries[0]->column); + auto kb = pair_kind_of(*group_entries[1]->column); + if (ka.has_value() && kb.has_value()) { + pair_int_fast_path_ = true; + int_key_kind_ = *ka; + int_key_kind_b_ = *kb; + const auto is_32_bit = [](IntKeyKind k) { + return k == IntKeyKind::Cat || k == IntKeyKind::Date; + }; + pair_packs_u64_ = is_32_bit(*ka) && is_32_bit(*kb); + } + } else if (group_entries.size() >= 3) { + // Three or more keys had no fast path at all: the branches above + // only recognise one key or two, so everything wider fell to + // `process_rows_generic`, which hashes a KeyCol tuple per row + // and hashes a Categorical as TEXT while doing it. If the whole + // key packs into a flat integer, `process_rows_packed` replaces + // that with one hash of a POD — and, because a packed key is + // something `try_discover_partitioned` can carry, threads the + // discovery too. + // + // The probe is discarded; the real plan is rebuilt per chunk, + // since a Categorical's remap is only valid for its own chunk. + packed_fast_path_ = encoder_.build_packed_key(group_entries).has_value(); + } + initialized_ = true; + } else { + for (std::size_t i = 0; i < n_aggs_; ++i) { + if (plan_[i].func == ir::AggFunc::Count) { + continue; + } + const ExprType kind = expr_type_for_column(*agg_entries[i]->column); + if (kind != plan_[i].kind) { + return "HashAggregateState: aggregate column type changed across chunks"; + } + } + for (std::size_t i = 0; i < group_entries.size(); ++i) { + if (group_entries[i]->column->index() != group_templates_[i].index()) { + return "HashAggregateState: group-by column type changed across chunks"; + } + } + } + + const std::size_t rows = chunk.rows(); + + // Global aggregate (`select { … }` with no `by`). Every row belongs to + // group 0, so the generic path below was running a hash probe per row + // against an EMPTY key just to rediscover that. Accumulate straight + // into the single group, and — since the groups are independent of row + // order — fan the row range out across workers. + if (group_entries.empty()) { + auto error = process_rows_ungrouped(agg_entries, rows); + if (!error.has_value()) { + publish_fused_accumulation(); + } + return error; + } + // A fast-path index records only raw values/codes. It therefore cannot + // distinguish a later null from that value's zero/code representation. + // Parquet commonly omits an all-valid row group's bitmap, so this is a + // real streaming transition rather than a schema change visible in the + // first chunk. + // + // Every fast path stores its groups' raw values in a form the generic + // `KeyRowIndex` can be reseeded from -- `int_order_`/`str_order_` + // directly, `cat_order_`/`multi_cat_codes_flat_` via the dictionary + // `group_templates_` still holds, `pair_order_` via both, and the + // packed path's `group_order_` is already boxed `Key`s (see + // `migrate_packed_fast_path_to_generic`). Migrating only rebuilds the + // key->gid lookup; the accumulated `flat_slots_`/`scratch_` those gids + // already own are untouched, and this chunk then runs the generic path + // below like any other. + if ((cat_fast_path_ || str_fast_path_ || int_fast_path_ || pair_int_fast_path_ || + packed_fast_path_) && + std::ranges::any_of(group_entries, [](const ColumnEntry* entry) { + return entry->validity.has_value(); + })) { + if (cat_fast_path_) { + migrate_cat_fast_path_to_generic(group_entries.size()); + } else if (int_fast_path_) { + migrate_int_fast_path_to_generic(); + } else if (str_fast_path_) { + migrate_str_fast_path_to_generic(); + } else if (pair_int_fast_path_) { + migrate_pair_int_fast_path_to_generic(); + } else if (packed_fast_path_) { + migrate_packed_fast_path_to_generic(); + } + } + if (cat_fast_path_) { + return process_rows_cat(group_entries, agg_entries, rows); + } + if (str_fast_path_) { + return process_rows_str(group_entries, agg_entries, rows); + } + if (int_fast_path_) { + return process_rows_int(group_entries, agg_entries, rows); + } + if (pair_int_fast_path_) { + return process_rows_int_pair(group_entries, agg_entries, rows); + } + if (packed_fast_path_ && rows != 0) { + auto plan = encoder_.build_packed_key(group_entries); + if (!plan.has_value()) { + // The shape was packable when the first chunk fixed the path, + // so this is an unsupported mid-stream key-layout transition. + return "HashAggregateState: group-by key column gained nulls across chunks"; + } + if (plan->width <= sizeof(std::uint64_t)) { + return process_rows_packed(group_entries, agg_entries, plan->cols, rows, packed64_); + } + if (plan->width <= sizeof(PackedKeyEncoder::Packed128)) { + return process_rows_packed(group_entries, agg_entries, plan->cols, rows, + packed128_); + } + return process_rows_packed(group_entries, agg_entries, plan->cols, rows, packed256_); + } + return process_rows_generic(group_entries, agg_entries, rows); + } + + auto process_rows_str(const std::vector& group_entries, + const std::vector& agg_entries, std::size_t rows) + -> std::optional { + const auto& col = std::get>(*group_entries[0]->column); + const char* src_chars = col.chars_data(); + const std::uint32_t* src_off = col.offsets_data(); + + gids_buf_.resize(rows); + auto* gids = gids_buf_.data(); + + // High-cardinality string keys are where this path pays: discovery is + // the serial half, and a string group-by has nothing else to hide it + // behind. Probing with a view keeps the owning copy per GROUP, as the + // serial loop below does. + const auto key_at = [&](std::size_t row) -> std::string_view { + return std::string_view{src_chars + src_off[row], src_off[row + 1] - src_off[row]}; + }; + if (try_discover_partitioned( + key_at, rows, gids, str_partitions_, [&](std::size_t n) { str_order_.resize(n); }, + [&](const std::string& key, std::uint32_t gid, std::size_t) { + str_order_[gid] = key; + }, + kDefaultPartitionMinRows, + [&](std::uint32_t gid) -> std::string_view { return str_order_[gid]; })) { + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + // Run-length shortcut: sorted or chunked CSV often has adjacent + // repeats; skip the hash lookup when the key matches the previous row. + std::string_view prev_key; + std::uint32_t prev_gid = std::numeric_limits::max(); + for (std::size_t row = 0; row < rows; ++row) { + const std::string_view key{src_chars + src_off[row], src_off[row + 1] - src_off[row]}; + std::uint32_t gid{}; + if (key == prev_key) { + gid = prev_gid; + } else { + // Transparent lookup on string_view avoids constructing a + // std::string per probe. Insertions pay one std::string + // construction per novel key — with libstdc++'s 15-char SSO, + // 11-char user_id strings stay inline (no heap alloc). + auto it = str_index_.find(key); + if (it == str_index_.end()) { + gid = static_cast(n_groups_); + str_index_.emplace(std::string(key), gid); + str_order_.emplace_back(key); + ++n_groups_; + size_group_arrays(); + } else { + gid = static_cast(it->second); + } + prev_key = key; + prev_gid = gid; + } + gids[row] = gid; + } + + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + // Single fixed-width-integer key: probe a value -> gid map directly, the way + // process_rows_str does for strings. Date/Timestamp are read as their raw + // integer (days / nanos), which is order- and equality-faithful. + auto process_rows_int(const std::vector& group_entries, + const std::vector& agg_entries, std::size_t rows) + -> std::optional { + const ColumnValue& key_col = *group_entries[0]->column; + const std::int64_t* i64 = nullptr; + const Date* dates = nullptr; + const Timestamp* stamps = nullptr; + switch (int_key_kind_) { + case IntKeyKind::Int64: + i64 = std::get>(key_col).data(); + break; + case IntKeyKind::Date: + dates = std::get>(key_col).data(); + break; + case IntKeyKind::Ts: + stamps = std::get>(key_col).data(); + break; + case IntKeyKind::Cat: + // A lone Categorical key never selects this path: it is + // all-Categorical by definition, so `cat_fast_path_` claims it + // and dispatches first. Only the pair path admits `Cat`. + return "HashAggregateState: categorical key on the single-int path"; + } + const auto key_at = [&](std::size_t row) -> std::int64_t { + switch (int_key_kind_) { + case IntKeyKind::Int64: + return i64[row]; + case IntKeyKind::Date: + return dates[row].days; + case IntKeyKind::Ts: + return stamps[row].nanos; + case IntKeyKind::Cat: + break; + } + return 0; + }; + + // The q18 shape (one Int64 key and one Double sum) is a streaming sink, + // not a sequence of per-chunk fork/join pipelines. Each chunk becomes + // one independent hot-table task; the caller immediately pulls the + // next chunk, and all tasks join once at end-of-stream. Besides removing + // the three barriers per chunk, this keeps the common clustered key in + // a 4096-slot cache-resident reduction and sends only cold/pre-aggregated + // records to the persistent partition maps. + if (try_async_hot_int_sum(group_entries[0]->column, agg_entries, rows)) { + publish_fused_accumulation(); + return std::nullopt; + } + + gids_buf_.resize(rows); + auto* gids = gids_buf_.data(); + + // A non-null First value is fixed at the same row that creates its + // group. Record it during discovery and omit the later all-row scan. + // Group-key reduction turns q10's six descriptive keys into exactly + // this shape; scanning 229k rows once per carried field was redundant. + if (discovery_first_eligible_.empty()) { + discovery_first_eligible_.resize(n_aggs_, 0U); + for (std::size_t a = 0; a < n_aggs_; ++a) { + discovery_first_eligible_[a] = plan_[a].func == ir::AggFunc::First ? 1U : 0U; + } + } + std::vector discovery_first(n_aggs_, 0U); + bool has_discovery_first = false; + if (std::getenv("IBEX_DISABLE_DISCOVERY_FIRST") == nullptr) { + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (discovery_first_eligible_[a] == 0U) { + continue; + } + if (agg_entries[a]->validity.has_value()) { + // A group may still be waiting for its first non-null + // value. Keep this field on the ordinary scan for every + // later chunk too, even if that later chunk has no nulls. + discovery_first_eligible_[a] = 0U; + continue; + } + discovery_first[a] = 1U; + has_discovery_first = true; + } + } + const std::size_t groups_before = n_groups_; + std::vector first_rows; + + // Partition-owned accumulation, the same shape the PairIntKey path + // above takes. It fuses discovery and the sum/count into one pass over + // partition-local state, so the global first-occurrence numbering -- + // and with it the whole second `accumulate_gids` scan of every row -- + // is deferred to a single merge over GROUPS at emission. + // + // `int_order_` holds the raw key whatever `int_key_kind_` is; the emit + // side reconstructs Date/Timestamp/Categorical from it exactly as it + // does for the ordinary int path, so this needs no kind-specific arm. + if (try_owned>( + key_at, rows, gids, agg_entries, owned_int_partitions_, kIntOwnedMinRows)) { + publish_fused_accumulation(); + return std::nullopt; + } + + if (try_discover_partitioned>( + key_at, rows, gids, int_partitions_, + [&](std::size_t n) { + int_order_.resize(n); + if (has_discovery_first) { + first_rows.resize(n - groups_before); + } + }, + [&](std::int64_t key, std::uint32_t gid, std::size_t row) { + int_order_[gid] = key; + if (has_discovery_first) { + first_rows[gid - groups_before] = row; + } + }, + kDefaultPartitionMinRows, [&](std::uint32_t gid) { return int_order_[gid]; })) { + if (has_discovery_first) { + seed_discovery_first(groups_before, first_rows, agg_entries, discovery_first); + } + publish_discovered(agg_entries, rows, has_discovery_first ? &discovery_first : nullptr); + return std::nullopt; + } + + // Run-length shortcut, as in the string path: sorted/chunked input often + // repeats the key, so skip the map lookup when it matches the last row. + std::int64_t prev_key = 0; + std::uint32_t prev_gid = std::numeric_limits::max(); + bool have_prev = false; + for (std::size_t row = 0; row < rows; ++row) { + const std::int64_t key = key_at(row); + std::uint32_t gid{}; + if (have_prev && key == prev_key) { + gid = prev_gid; + } else { + auto it = int_index_.find(key); + if (it == int_index_.end()) { + gid = static_cast(n_groups_); + int_index_.emplace(key, gid); + int_order_.push_back(key); + ++n_groups_; + size_group_arrays(); + if (has_discovery_first) { + first_rows.push_back(row); + } + } else { + gid = it->second; + } + prev_key = key; + prev_gid = gid; + have_prev = true; + } + gids[row] = gid; + } + + if (has_discovery_first) { + seed_discovery_first(groups_before, first_rows, agg_entries, discovery_first); + } + publish_discovered(agg_entries, rows, has_discovery_first ? &discovery_first : nullptr); + return std::nullopt; + } + + static constexpr std::size_t kOwnedHotSlots = 4096; + + struct OwnedHotRecord { + std::int64_t key = 0; + std::uint64_t first_row = 0; + AggSlotCore slot; + }; + + struct OwnedHotChunk { + std::shared_ptr key_column; + std::shared_ptr sum_column; + std::optional sum_validity; + std::uint64_t row_base = 0; + std::size_t rows = 0; + std::size_t part_count = 0; + std::vector> records_by_partition; + std::optional error; + }; + + struct OwnedHotSlot { + std::uint32_t tag = std::numeric_limits::max(); + std::uint32_t last_access_tag = std::numeric_limits::max(); + std::uint32_t record = std::numeric_limits::max(); + }; + + [[nodiscard]] static auto owned_hot_hash(std::int64_t key) noexcept -> std::uint64_t { + // robin_hood's Int64 hash preserves too much of a sequential key's bit + // pattern for a high-bit fixed table. SplitMix's finalizer gives both + // candidate slots and the tag independent-looking bits at tiny cost. + auto x = static_cast(key); + x ^= x >> 30U; + x *= 0xbf58476d1ce4e5b9ULL; + x ^= x >> 27U; + x *= 0x94d049bb133111ebULL; + x ^= x >> 31U; + return x; + } + + static void process_owned_hot_chunk(OwnedHotChunk& job) noexcept { + try { + const auto* keys = std::get>(*job.key_column).data(); + const auto* values = std::get>(*job.sum_column).data(); + const ValidityBitmap* validity = + job.sum_validity.has_value() ? &*job.sum_validity : nullptr; + constexpr std::uint32_t kEmpty = std::numeric_limits::max(); + constexpr unsigned kShift = 64U - 12U; + constexpr std::uint64_t kH2Mult = 0xf1357aea2e62a9c5ULL; + + std::array table{}; + std::vector records; + records.reserve(std::max(kOwnedHotSlots, job.rows / 3)); + std::size_t filled = 0; + std::uint64_t prng = 0; + std::int64_t last_key = 0; + std::uint32_t last_record = kEmpty; + bool have_last = false; + + const auto append_record = [&](std::size_t row) -> std::uint32_t { + const auto index = static_cast(records.size()); + OwnedHotRecord record; + record.key = keys[row]; + record.first_row = job.row_base + row; + if (validity == nullptr || (*validity)[row]) { + record.slot.double_value = values[row]; + record.slot.mark_present(); + } + records.push_back(record); + return index; + }; + const auto update_record = [&](std::uint32_t record, std::size_t row) { + if (validity == nullptr || (*validity)[row]) { + auto& slot = records[record].slot; + slot.double_value += values[row]; + slot.mark_present(); + } + }; + + for (std::size_t row = 0; row < job.rows; ++row) { + const std::int64_t key = keys[row]; + // HashKeys in Polars computes hashes as a column kernel before + // probing. Here keys arrive as raw Int64s, and q18's useful + // locality is specifically runs of the same key. Retain the + // most recent hot/cold record so the other rows in a run avoid + // both the SplitMix hash and the two fixed-table probes. + if (have_last && key == last_key) { + update_record(last_record, row); + continue; + } + const std::uint64_t hash = owned_hot_hash(key); + const auto tag = static_cast(hash); + const auto h1 = static_cast(hash >> kShift); + const auto h2 = static_cast((hash * kH2Mult) >> kShift); + auto& s1 = table[h1]; + auto& s2 = table[h2]; + + if (s1.tag == tag && s1.record != kEmpty && records[s1.record].key == key) { + s1.last_access_tag = tag; + update_record(s1.record, row); + last_key = key; + last_record = s1.record; + have_last = true; + continue; + } + if (s2.tag == tag && s2.record != kEmpty && records[s2.record].key == key) { + s2.last_access_tag = tag; + update_record(s2.record, row); + last_key = key; + last_record = s2.record; + have_last = true; + continue; + } + + if (filled < kOwnedHotSlots) { + OwnedHotSlot* empty = s1.record == kEmpty ? &s1 : nullptr; + if (empty == nullptr && s2.record == kEmpty) { + empty = &s2; + } + if (empty != nullptr) { + empty->tag = tag; + empty->last_access_tag = tag; + empty->record = append_record(row); + last_key = key; + last_record = empty->record; + have_last = true; + ++filled; + continue; + } + } + + // Polars' second chance: a miss first marks one candidate as + // recently considered and stays cold. Seeing the same tag + // again earns admission, evicting that candidate's mapping; + // its record already contains the complete pre-aggregate. + OwnedHotSlot& chosen = (prng >> 63U) != 0 ? s1 : s2; + prng += hash; + if (chosen.last_access_tag == tag) { + chosen.tag = tag; + chosen.last_access_tag = tag; + chosen.record = append_record(row); + last_record = chosen.record; + } else { + chosen.last_access_tag = tag; + last_record = append_record(row); + } + last_key = key; + have_last = true; + } + + // Records were appended at their first source row and only updated + // in place, so this stable routing preserves global first-seen + // order without a histogram/scatter phase or a sort. + job.records_by_partition.resize(job.part_count); + std::vector counts(job.part_count, 0); + const robin_hood::hash partition_hash; + const std::size_t mask = job.part_count - 1; + for (const auto& record : records) { + ++counts[partition_hash(record.key) & mask]; + } + for (std::size_t p = 0; p < job.part_count; ++p) { + job.records_by_partition[p].reserve(counts[p]); + } + for (auto& record : records) { + const std::size_t p = partition_hash(record.key) & mask; + job.records_by_partition[p].push_back(record); + } + + // Release decoded buffers as soon as this task is done. The job's + // compact pre-aggregates remain until the one final cold merge. + job.key_column.reset(); + job.sum_column.reset(); + job.sum_validity.reset(); + } catch (const std::exception& error) { + job.error = "async hot aggregate: " + std::string(error.what()); + } catch (...) { + job.error = "async hot aggregate: non-standard worker exception"; + } + } + + auto try_async_hot_int_sum(const std::shared_ptr& key_column, + const std::vector& agg_entries, std::size_t rows) + -> bool { + if (!owned_async_hot_mode_) { + if (std::getenv("IBEX_DISABLE_OWNED_PAIR_AGG") != nullptr || + std::getenv("IBEX_DISABLE_ASYNC_HOT_AGG") != nullptr || n_groups_ > 0 || + partitioned_active_ || owned_mode_ || n_aggs_ != 1 || + plan_[0].func != ir::AggFunc::Sum || plan_[0].kind != ExprType::Double || + int_key_kind_ != IntKeyKind::Int64 || scratch_stride_ != 0 || exec_ == nullptr || + on_worker_pool_thread() || std::max(rows_offered_, rows) < kIntOwnedMinRows) { + return false; + } + // As `try_owned`: fan-out permission and the worker cap are the + // plan's Discovery node (src/runtime/PARALLELISM.md); + // `kIntOwnedMinRows` stays as this specialization's admission gate. + if (par_.discovery.decline != physical::FanOutDecline::None || + par_.discovery.worker_cap < 2) { + return false; + } + auto& pool = process_worker_pool(); + const std::size_t workers = par_.discovery.worker_cap; + owned_async_part_count_ = 1; + while (owned_async_part_count_ * 2 <= workers) { + owned_async_part_count_ *= 2; + } + owned_int_partitions_.resize(owned_async_part_count_); + owned_async_group_.emplace(pool.task_group()); + owned_async_hot_mode_ = true; + owned_mode_ = true; + note_partition_fanout(); + } + + const ColumnEntry& agg0 = *agg_entries[0]; + auto job = std::make_unique(); + job->key_column = key_column; + job->sum_column = agg0.column; + if (agg0.validity.has_value()) { + job->sum_validity = *agg0.validity; + } + job->row_base = owned_rows_seen_; + job->rows = rows; + job->part_count = owned_async_part_count_; + auto* const raw_job = job.get(); + owned_async_jobs_.push_back(std::move(job)); + // Engaged since the block above either emplaced it or `owned_async_hot_mode_` + // was already set (the two only ever change together). + if (!owned_async_group_.has_value()) { + invariant_violation("async hot aggregate: task group missing while accepting chunks"); + } + owned_async_group_->submit([raw_job] { process_owned_hot_chunk(*raw_job); }); + owned_rows_seen_ += rows; + return true; + } + + /// Production ownership threshold for the narrow PairIntKey path below, + /// backed by a synthetic row/cardinality sweep (32k/64k/128k/262144 rows + /// x low/high cardinality, 8 cores, 6 interleaved rounds): 32k showed no + /// reliable win (3/6 wins, ~0%), 64k was the first point with a + /// consistent, real one (6/6 wins, -6% to -10%), and 128k/262144 stayed + /// positive. Deliberately NOT `kDefaultPartitionMinRows` (262144, the + /// threshold `try_discover_partitioned` uses): that value was tuned for a + /// different mechanism (discovery only, no fused accumulation, no + /// deferred merge) and is not evidence for where THIS path's overhead + /// breaks even -- q20's own chunks (~150k rows) sit between the two. + static constexpr std::size_t kPairOwnedMinRows = 1U << 16U; // 65536 + + /// Same gate for the single-Int64-key slice. Held at the pair path's value + /// until the sweep below says otherwise -- the mechanism is identical and + /// its break-even has no reason to differ by more than the key's own probe + /// cost, which is lower, not higher. + static constexpr std::size_t kIntOwnedMinRows = 1U << 16U; // 65536 + + /// Production PairIntKey ownership path (TPC-H q20's + /// `by { l_partkey, l_suppkey }` is the motivating shape; validated + /// there at -16.5%, 8/8 paired wins, 8 cores, vs. a q18/Int64 prototype + /// that measured only -7.6%, was never promoted, and has since been + /// removed -- see plans/parallelism-overview.md). Deliberately narrow: + /// + /// - Exactly one aggregate, Sum(Double) or Count. q18 and q20 both only + /// ever exercise one, so nothing measures whether row-wise fusion beats + /// partition-outer/aggregate-outer accumulation once a query carries + /// several -- widen only after that shape is actually benchmarked. + /// - Row-wise fusion only: with exactly one aggregate a second full row + /// scan can only add cost, never locality it does not already have. + /// - No env-var mode selector: this runs whenever eligible, the same way + /// `try_discover_partitioned` has no toggle either. `IBEX_DISABLE_ + /// OWNED_PAIR_AGG=1` is a kill switch for the unusual case that needs + /// one, not a normal control surface. + template + auto try_owned(const KeyAt& key_at, std::size_t rows, std::uint32_t* gids, + const std::vector& agg_entries, Partitions& partitions, + std::size_t min_rows) -> bool { + if (std::getenv("IBEX_DISABLE_OWNED_PAIR_AGG") != nullptr) { + return false; + } + if (!owned_mode_) { + if (n_groups_ > 0 || partitioned_active_) { + return false; + } + if (n_aggs_ != 1) { + return false; + } + if (plan_[0].func != ir::AggFunc::Sum && plan_[0].func != ir::AggFunc::Count) { + return false; + } + if (plan_[0].func == ir::AggFunc::Sum && plan_[0].kind != ExprType::Double) { + return false; + } + if (scratch_stride_ != 0) { + return false; + } + if (exec_ == nullptr || on_worker_pool_thread()) { + return false; + } + // Fan-out permission and the worker cap are the plan's + // (src/runtime/PARALLELISM.md); `decline != None` folds in + // `!exec_->can_fan_out()`. `min_rows` stays here: it is the owned + // strategy's own "is the specialization worth it" gate, lower than + // Discovery's radix floor, and an operator-resolved choice like + // the join's build orientation. + if (par_.discovery.decline != physical::FanOutDecline::None || + par_.discovery.worker_cap < 2) { + return false; + } + if (std::max(rows_offered_, rows) < min_rows) { + return false; + } + } + + auto& pool = process_worker_pool(); + const std::size_t workers = par_.discovery.worker_cap; + std::size_t part_count = 1; + while (part_count * 2 <= workers) { + part_count *= 2; + } + const std::uint64_t part_mask = part_count - 1; + if (partitions.size() < part_count) { + partitions.resize(part_count); + } + note_partition_fanout(); + + // A clustered count key should not pay the partition/scatter/hash + // pipeline once per ROW. Compress contiguous equal-key runs first and + // carry their lengths as count weights. This is exact for arbitrary + // input order: non-contiguous runs still meet in the exact hash + // fallback, while sorted inputs such as TPC-H lineitem reduce the + // expensive part of the pipeline by roughly their mean run length. + // Sample before committing because all-unique keys would only add two + // equality passes and retain the original item count. + // NOLINTNEXTLINE(misc-const-correctness) -- mutated in the int64 instantiation below + [[maybe_unused]] bool compress_runs = false; + if constexpr (std::is_same_v) { + compress_runs = owned_ordered_run_mode_; + if (!compress_runs && !owned_mode_ && plan_[0].func == ir::AggFunc::Count && + rows >= 64 && std::getenv("IBEX_DISABLE_ORDERED_RUN_AGG") == nullptr) { + const std::size_t sampled = std::min(rows, 8192); + std::size_t repeats = 0; + Key previous = key_at(0); + for (std::size_t row = 1; row < sampled; ++row) { + const Key key = key_at(row); + repeats += key == previous ? 1 : 0; + previous = key; + } + compress_runs = repeats * 2 >= sampled - 1; + } + } + + const std::size_t source_ranges = workers; + const std::size_t source_grain = (rows + source_ranges - 1) / source_ranges; + + // Run compression only ever engages for a clustered single-Int64 Count + // (the `if constexpr` above is the only writer of `compress_runs`). + // Two parallel passes, no scratch: pass 1 counts runs per range so the + // per-range output offsets are contiguous, pass 2 emits (key, length) + // straight into `owned_ordered_run_{keys,counts}_`. A third pass to + // dereference anchor rows -- and the `owned_run_rows_/_lengths_` arrays + // it read -- used to sit between them; folding it into pass 2 drops one + // pool barrier per chunk, which is q21's `count() by l_orderkey` hot + // path (~23 chunks, this was 3 barriers each). + if constexpr (std::is_same_v) { + if (compress_runs) { + std::vector run_offsets(source_ranges + 1, 0); + { + auto batch = pool.submit(source_ranges, [&](std::size_t r) { + const std::size_t begin = r * source_grain; + const std::size_t end = std::min(rows, begin + source_grain); + if (begin >= end) { + return; + } + std::size_t runs = 1; + Key previous = key_at(begin); + for (std::size_t row = begin + 1; row < end; ++row) { + const Key key = key_at(row); + runs += key == previous ? 0 : 1; + previous = key; + } + run_offsets[r + 1] = runs; + }); + batch.wait(); + } + for (std::size_t r = 0; r < source_ranges; ++r) { + run_offsets[r + 1] += run_offsets[r]; + } + const std::size_t items = run_offsets.back(); + const std::size_t old_runs = owned_ordered_run_keys_.size(); + owned_ordered_run_keys_.resize(old_runs + items); + owned_ordered_run_counts_.resize(old_runs + items); + { + auto batch = pool.submit(source_ranges, [&](std::size_t r) { + const std::size_t begin = r * source_grain; + const std::size_t end = std::min(rows, begin + source_grain); + if (begin >= end) { + return; + } + std::size_t out = old_runs + run_offsets[r]; + std::size_t anchor = begin; + Key previous = key_at(begin); + for (std::size_t row = begin + 1; row < end; ++row) { + const Key key = key_at(row); + if (key != previous) { + owned_ordered_run_keys_[out] = previous; + owned_ordered_run_counts_[out] = row - anchor; + ++out; + anchor = row; + previous = key; + } + } + owned_ordered_run_keys_[out] = previous; + owned_ordered_run_counts_[out] = end - anchor; + }); + batch.wait(); + } + if (owned_ordered_runs_nondecreasing_) { + const std::size_t from = old_runs == 0 ? 1 : old_runs; + for (std::size_t i = from; i < old_runs + items; ++i) { + if (owned_ordered_run_keys_[i] < owned_ordered_run_keys_[i - 1]) { + owned_ordered_runs_nondecreasing_ = false; + break; + } + } + } + owned_rows_seen_ += rows; + owned_mode_ = true; + owned_ordered_run_mode_ = true; + return true; + } + } + + const std::size_t ranges = workers; + const std::size_t grain = (rows + ranges - 1) / ranges; + part_of_row_.resize(rows); + std::vector counts(ranges * part_count, 0); + { + auto batch = pool.submit(ranges, [&](std::size_t r) { + const std::size_t begin = r * grain; + const std::size_t end = std::min(rows, begin + grain); + std::size_t* row_counts = counts.data() + (r * part_count); + Hash hasher; + for (std::size_t row = begin; row < end; ++row) { + const auto part = static_cast(hasher(key_at(row)) & part_mask); + part_of_row_[row] = part; + ++row_counts[part]; + } + }); + batch.wait(); + } + std::vector offsets(ranges * part_count, 0); + std::vector part_begin(part_count + 1, 0); + { + std::size_t running = 0; + for (std::size_t p = 0; p < part_count; ++p) { + part_begin[p] = running; + for (std::size_t r = 0; r < ranges; ++r) { + offsets[(r * part_count) + p] = running; + running += counts[(r * part_count) + p]; + } + } + part_begin[part_count] = running; + } + scatter_rows_.resize(rows); + { + auto batch = pool.submit(ranges, [&](std::size_t r) { + const std::size_t begin = r * grain; + const std::size_t end = std::min(rows, begin + grain); + std::size_t* cursor = offsets.data() + (r * part_count); + for (std::size_t row = begin; row < end; ++row) { + scatter_rows_[cursor[part_of_row_[row]]++] = row; + } + }); + batch.wait(); + } + + std::vector sum_cols(n_aggs_, nullptr); + std::vector sum_validity(n_aggs_, nullptr); + std::vector is_count(n_aggs_, 0); + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (plan_[a].func == ir::AggFunc::Count) { + is_count[a] = 1; + continue; + } + const ColumnEntry& entry = *agg_entries[a]; + sum_cols[a] = std::get>(*entry.column).data(); + sum_validity[a] = entry.validity.has_value() ? &*entry.validity : nullptr; + } + + const std::uint64_t row_base = owned_rows_seen_; + { + std::atomic cursor{0}; + auto batch = pool.submit(std::min(workers, part_count), [&](std::size_t) { + for (std::size_t p = cursor.fetch_add(1, std::memory_order_relaxed); p < part_count; + p = cursor.fetch_add(1, std::memory_order_relaxed)) { + auto& partition = partitions[p]; + for (std::size_t i = part_begin[p]; i < part_begin[p + 1]; ++i) { + const std::size_t row = scatter_rows_[i]; + const Key key = key_at(row); + auto it = partition.index.find(key); + std::uint32_t local{}; + if (it == partition.index.end()) { + local = static_cast(partition.keys.size()); + partition.index.emplace(key, local); + partition.keys.push_back(key); + partition.first_rows.push_back(row_base + row); + partition.slots.resize((local + 1) * n_aggs_); + } else { + local = it->second; + } + gids[row] = local; + // n_aggs_ == 1 is enforced above -- this is a single + // slot update, not a loop over aggregates. Written as + // one, not unrolled, so a future widening to >1 + // aggregate (once actually measured, per the + // class-level comment) is a small diff here. + AggSlotCore& slot = partition.slots[local]; + if (is_count[0] != 0) { + ++slot.count; + } else if (sum_validity[0] == nullptr || (*sum_validity[0])[row]) { + slot.double_value += sum_cols[0][row]; + slot.mark_present(); + } + } + } + }); + batch.wait(); + } + + owned_rows_seen_ += rows; + owned_mode_ = true; + if (std::getenv("IBEX_AGG_PARTITION_DEBUG") != nullptr) { + ibex::formatting::print( + stderr, "[agg_owned] chunk rows={} part_count={} total_rows={}\n", rows, + partitions.size(), static_cast(owned_rows_seen_)); + } + return true; + } + + /// Walk every `owned_pair_partitions_` entry once, in first-occurrence + /// order (a P-way merge over `first_rows`, run once at final emission), + /// and populate `pair_order_`/`flat_slots_` -- the arrays + /// `build_output_chunk`'s `pair_int_fast_path_` branch already reads. + template + void finalize_owned(Partitions& partitions, const ResizeOrder& resize_order, + const StoreKey& store_key) { + if (owned_finalized_) { + return; + } + owned_finalized_ = true; + const std::size_t part_count = partitions.size(); + std::size_t total = 0; + for (const auto& partition : partitions) { + total += partition.keys.size(); + } + n_groups_ = total; + resize_order(total); + AggSlotCore* fs = flat_slots_.grow_uninitialized(total * n_aggs_).data(); + if (total == 0) { + return; + } + + // K-way merge of the `part_count` partition group-lists by `first_rows` + // into the output at ascending `g`. Every partition's `first_rows` is + // strictly ascending and the values are globally unique (they are row + // indices, and each row belongs to one partition), so this is a stable + // merge over a total order -- byte-identical however the segments below + // are split. + const auto merge_segment = [&](std::vector cur, + const std::vector& stop, std::size_t g) { + for (;;) { + std::size_t best = part_count; + std::uint64_t best_row = std::numeric_limits::max(); + for (std::size_t p = 0; p < part_count; ++p) { + if (cur[p] >= stop[p]) { + continue; + } + const std::uint64_t fr = partitions[p].first_rows[cur[p]]; + if (fr < best_row) { + best_row = fr; + best = p; + } + } + if (best == part_count) { + break; + } + const auto& partition = partitions[best]; + const std::size_t local = cur[best]; + store_key(g, partition.keys[local]); + for (std::size_t a = 0; a < n_aggs_; ++a) { + fs[(g * n_aggs_) + a] = partition.slots[(local * n_aggs_) + a]; + } + ++cur[best]; + ++g; + } + }; + + std::vector part_end(part_count); + for (std::size_t p = 0; p < part_count; ++p) { + part_end[p] = partitions[p].keys.size(); + } + + // For q18's 3M groups the serial merge is ~24M comparisons plus 3M + // slot copies -- tens of ms on the calling thread. Split the OUTPUT + // into per-worker rank ranges via merge-path co-ranking: since every + // `first_rows` value is unique, `sum_p lower_bound(first_rows_p, v)` + // steps by exactly one at each value and so equals any target rank at + // exactly one `v`. Each worker then merges the disjoint input slices + // between two frontiers into its disjoint output slice. + // The FinalOrdering node's worker ceiling and fan-out permission are the + // plan's (src/runtime/PARALLELISM.md); `part_count` and `total / 4096` + // stay here -- they need the group count discovery just produced. The + // `1U << 17U` group floor is this merge's own threshold, beside it. + std::size_t workers = 1; + if (exec_ != nullptr && par_.final_ordering.decline == physical::FanOutDecline::None && + !on_worker_pool_thread() && part_count >= 2 && total >= std::size_t{1} << 17U) { + workers = std::min({par_.final_ordering.worker_cap, part_count, total / 4096}); + } + + if (workers < 2) { + merge_segment(std::vector(part_count, 0), part_end, 0); + return; + } + note_finalize_fanout(); + + const std::uint64_t hi = owned_rows_seen_ + 1; + const auto frontier = [&](std::size_t rank) { + std::uint64_t lo = 0; + std::uint64_t high = hi; + while (lo < high) { + const std::uint64_t mid = lo + ((high - lo) / 2); + std::size_t sum = 0; + for (std::size_t p = 0; p < part_count; ++p) { + const auto& fr = partitions[p].first_rows; + sum += static_cast(std::lower_bound(fr.begin(), fr.end(), mid) - + fr.begin()); + } + if (sum < rank) { + lo = mid + 1; + } else { + high = mid; + } + } + std::vector off(part_count); + for (std::size_t p = 0; p < part_count; ++p) { + const auto& fr = partitions[p].first_rows; + off[p] = static_cast(std::lower_bound(fr.begin(), fr.end(), lo) - + fr.begin()); + } + return off; + }; + + std::vector> bounds(workers + 1); + bounds.front().assign(part_count, 0); + bounds.back() = part_end; + for (std::size_t w = 1; w < workers; ++w) { + bounds[w] = frontier(w * total / workers); + } + + auto batch = process_worker_pool().submit(workers, [&](std::size_t w) { + std::size_t g = 0; + for (std::size_t p = 0; p < part_count; ++p) { + g += bounds[w][p]; + } + merge_segment(bounds[w], bounds[w + 1], g); + }); + batch.wait(); + } + + /// Dispatch the deferred merge to whichever key the owned run filled. Only + /// one can be non-empty: the gate admits an owned run only before any group + /// exists, so a single operator commits to one key and keeps it. + void finalize_owned_active() { + if (owned_async_hot_mode_) { + finalize_owned_async_hot(); + } else if (owned_ordered_run_mode_) { + finalize_owned_ordered_runs(); + } else if (!owned_pair_partitions_.empty()) { + finalize_owned( + owned_pair_partitions_, [&](std::size_t n) { pair_order_.resize(n); }, + [&](std::size_t g, const PairIntKey& key) { + pair_order_[g] = {static_cast(key.first), + static_cast(key.second)}; + }); + } else if (!owned_int_partitions_.empty()) { + finalize_owned( + owned_int_partitions_, [&](std::size_t n) { int_order_.resize(n); }, + [&](std::size_t g, std::int64_t key) { int_order_[g] = key; }); + } + } + + /// Join the streaming hot-table tasks once, then let one worker own each + /// cold partition for its complete lifetime. There is no chunk-local + /// histogram, scatter, or accumulate barrier: chunks only publish tasks; + /// the two synchronization points here are whole-stream hot completion and + /// whole-stream cold completion. + void finalize_owned_async_hot() { + if (owned_finalized_) { + return; + } + owned_finalized_ = true; + if (!owned_async_group_.has_value()) { + invariant_violation("async hot aggregate join: task group already released"); + } + try { + owned_async_group_->wait(); + } catch (const std::exception& error) { + owned_async_error_ = "async hot aggregate join: " + std::string(error.what()); + return; + } catch (...) { + owned_async_error_ = "async hot aggregate join: non-standard worker exception"; + return; + } + for (const auto& job : owned_async_jobs_) { + if (job->error.has_value()) { + owned_async_error_ = *job->error; + return; + } + } + + auto& partitions = owned_int_partitions_; + const std::size_t part_count = owned_async_part_count_; + if (part_count >= 2) { + note_finalize_fanout(); + } + try { + auto batch = process_worker_pool().submit(part_count, [&](std::size_t p) { + auto& partition = partitions[p]; + std::size_t records = 0; + for (const auto& job : owned_async_jobs_) { + records += job->records_by_partition[p].size(); + } + + // This reserve runs concurrently per owner. Unlike the failed + // calling-thread reserve experiment, it neither serializes the + // partitions nor guesses from total input rows; the exact + // pre-aggregate count is a safe upper bound on distinct keys. + partition.index.reserve(records); + partition.keys.reserve(records); + partition.first_rows.reserve(records); + partition.slots.reserve(records); + + for (const auto& job : owned_async_jobs_) { + for (const auto& record : job->records_by_partition[p]) { + auto it = partition.index.find(record.key); + std::uint32_t local{}; + if (it == partition.index.end()) { + local = static_cast(partition.keys.size()); + partition.index.emplace(record.key, local); + partition.keys.push_back(record.key); + partition.first_rows.push_back(record.first_row); + partition.slots.push_back(record.slot); + } else { + local = it->second; + if (record.slot.present()) { + auto& slot = partition.slots[local]; + slot.double_value += record.slot.double_value; + slot.mark_present(); + } + } + } + } + }); + batch.wait(); + } catch (const std::exception& error) { + owned_async_error_ = "async cold aggregate: " + std::string(error.what()); + return; + } catch (...) { + owned_async_error_ = "async cold aggregate: non-standard worker exception"; + return; + } + + // Pre-aggregate payloads are no longer needed. Release them before the + // output arrays are allocated so peak memory is cold state + output, + // rather than cold state + every streamed record + output. + owned_async_jobs_.clear(); + owned_async_group_.reset(); + + // Reuse the already-parallel first-occurrence merge. `first_rows` in + // every cold partition is ascending because hot records are created at + // their first source row and jobs are consumed in input order. + owned_finalized_ = false; + finalize_owned( + partitions, [&](std::size_t n) { int_order_.resize(n); }, + [&](std::size_t g, std::int64_t key) { int_order_[g] = key; }); + } + + /// A clustered single-Int64 Count is summarized as contiguous runs while + /// chunks arrive. If the complete stream is nondecreasing, adjacent runs + /// are the final groups and no hash table is needed at all. If a later + /// chunk disproves ordering, merge the run summaries through a hash map at + /// emission; this preserves exact first-occurrence semantics without + /// retaining or replaying the input rows. + void finalize_owned_ordered_runs() { + if (owned_finalized_) { + return; + } + owned_finalized_ = true; + if (owned_ordered_run_keys_.empty()) { + n_groups_ = 0; + return; + } + + if (owned_ordered_runs_nondecreasing_) { + const bool ord_timing = std::getenv("IBEX_ORDERED_RUN_TIMING") != nullptr; + const auto ord_t0 = std::chrono::steady_clock::now(); + const std::size_t run_count = owned_ordered_run_keys_.size(); + const std::int64_t* const rk = owned_ordered_run_keys_.data(); + const std::size_t* const rc = owned_ordered_run_counts_.data(); + + // How many workers can help. The build loop below is a segmented + // reduction over `run_count` sorted (key, count) runs: keys are + // nondecreasing so a group boundary is just `rk[i] != rk[i-1]`. + // Each worker owns a contiguous run slice; a key straddling a slice + // boundary has its leading partial count carried back to the group + // the previous worker finished (at most `workers - 1` fixups). + // Ceiling and permission from the plan; `run_count / 8192` and this + // path's own `1U << 16U` run floor stay here (the ordered-run merge + // is a strategy specialization, floor beside its code). + std::size_t workers = 1; + if (exec_ != nullptr && par_.final_ordering.decline == physical::FanOutDecline::None && + !on_worker_pool_thread() && run_count >= (std::size_t{1} << 16U) && + std::getenv("IBEX_DISABLE_PARALLEL_ORDERED_MERGE") == nullptr) { + workers = std::min({par_.final_ordering.worker_cap, run_count / 8192}); + } + + std::size_t total = 0; + if (workers < 2) { + total = run_count == 0 ? 0 : 1; + for (std::size_t i = 1; i < run_count; ++i) { + total += rk[i] == rk[i - 1] ? 0 : 1; + } + n_groups_ = total; + int_order_.resize(total); + AggSlotCore* slots = flat_slots_.grow_uninitialized(total).data(); + if (total != 0) { + std::size_t group = 0; + int_order_[0] = rk[0]; + slots[0] = AggSlotCore{}; + slots[0].count = static_cast(rc[0]); + for (std::size_t i = 1; i < run_count; ++i) { + if (rk[i] != int_order_[group]) { + ++group; + int_order_[group] = rk[i]; + slots[group] = AggSlotCore{}; + } + slots[group].count += static_cast(rc[i]); + } + } + } else { + note_finalize_fanout(); + auto& pool = process_worker_pool(); + const std::size_t grain = (run_count + workers - 1) / workers; + std::vector local_groups(workers, 0); + std::vector boundary_open(workers, 0); + { + auto batch = pool.submit(workers, [&](std::size_t w) { + const std::size_t begin = w * grain; + const std::size_t end = std::min(run_count, begin + grain); + if (begin >= end) { + return; + } + std::size_t g = 0; + for (std::size_t i = begin; i < end; ++i) { + if (i == 0 || rk[i] != rk[i - 1]) { + ++g; + } + } + local_groups[w] = g; + boundary_open[w] = (begin > 0 && rk[begin] == rk[begin - 1]) ? 1 : 0; + }); + batch.wait(); + } + std::vector group_base(workers + 1, 0); + for (std::size_t w = 0; w < workers; ++w) { + group_base[w + 1] = group_base[w] + local_groups[w]; + } + total = group_base[workers]; + n_groups_ = total; + int_order_.resize(total); + AggSlotCore* slots = flat_slots_.grow_uninitialized(total).data(); + std::vector carry(workers, 0); + { + auto batch = pool.submit(workers, [&](std::size_t w) { + const std::size_t begin = w * grain; + const std::size_t end = std::min(run_count, begin + grain); + if (begin >= end) { + return; + } + std::size_t i = begin; + if (boundary_open[w] != 0) { + const std::int64_t k0 = rk[begin]; + std::int64_t c = 0; + while (i < end && rk[i] == k0) { + c += static_cast(rc[i]); + ++i; + } + carry[w] = c; + } + std::size_t g = group_base[w]; + while (i < end) { + const std::int64_t k = rk[i]; + std::int64_t c = 0; + while (i < end && rk[i] == k) { + c += static_cast(rc[i]); + ++i; + } + int_order_[g] = k; + slots[g] = AggSlotCore{}; + slots[g].count = c; + ++g; + } + }); + batch.wait(); + } + for (std::size_t w = 0; w < workers; ++w) { + if (boundary_open[w] != 0) { + slots[group_base[w] - 1].count += carry[w]; + } + } + } + + if (ord_timing) { + const auto ms = std::chrono::duration( + std::chrono::steady_clock::now() - ord_t0) + .count(); + ibex::formatting::print(stderr, + "[ord_run] finalize nondecreasing runs={} groups={} " + "workers={} {}ms\n", + run_count, total, workers, ms); + } + return; + } + + robin_hood::unordered_flat_map index; + std::vector counts; + for (std::size_t i = 0; i < owned_ordered_run_keys_.size(); ++i) { + const std::int64_t key = owned_ordered_run_keys_[i]; + auto it = index.find(key); + std::uint32_t group{}; + if (it == index.end()) { + group = static_cast(int_order_.size()); + index.emplace(key, group); + int_order_.push_back(key); + counts.push_back(0); + } else { + group = it->second; + } + counts[group] += static_cast(owned_ordered_run_counts_[i]); + } + n_groups_ = int_order_.size(); + AggSlotCore* slots = flat_slots_.grow_uninitialized(n_groups_).data(); + for (std::size_t group = 0; group < n_groups_; ++group) { + slots[group] = AggSlotCore{}; + slots[group].count = counts[group]; + } + } + + // Two fixed-width-integer keys, grouped as one composite. Mirrors + // process_rows_int exactly, packing (key_a, key_b) into a two-word key so + // a single hash probe replaces the generic path's per-key Key comparison. + auto process_rows_int_pair(const std::vector& group_entries, + const std::vector& agg_entries, std::size_t rows) + -> std::optional { + // Bind the key column's buffer once, the way `process_rows_int` does. + // Reading it through `std::get` per row costs a variant index check per + // key per row and re-derives the pointer every time, which on 8M rows + // over two keys was most of this loop. + struct RawKeyReader { + const std::int64_t* i64 = nullptr; + const Date* dates = nullptr; + const Timestamp* stamps = nullptr; + const Column::code_type* codes = nullptr; + IntKeyKind kind = IntKeyKind::Int64; + + [[nodiscard]] auto operator()(std::size_t row) const -> std::int64_t { + switch (kind) { + case IntKeyKind::Int64: + return i64[row]; + case IntKeyKind::Date: + return dates[row].days; + case IntKeyKind::Ts: + return stamps[row].nanos; + case IntKeyKind::Cat: + return codes[row]; + } + return 0; + } + }; + const auto bind_reader = [](const ColumnValue& col, IntKeyKind kind) -> RawKeyReader { + RawKeyReader reader; + reader.kind = kind; + switch (kind) { + case IntKeyKind::Int64: + reader.i64 = std::get>(col).data(); + break; + case IntKeyKind::Date: + reader.dates = std::get>(col).data(); + break; + case IntKeyKind::Ts: + reader.stamps = std::get>(col).data(); + break; + case IntKeyKind::Cat: + reader.codes = std::get>(col).codes_data(); + break; + } + return reader; + }; + const auto key_a_at = bind_reader(*group_entries[0]->column, int_key_kind_); + const auto key_b_at = bind_reader(*group_entries[1]->column, int_key_kind_b_); + const auto pack = [](std::int64_t a, std::int64_t b) -> PairIntKey { + return {.first = static_cast(a), + .second = static_cast(b)}; + }; + + gids_buf_.resize(rows); + auto* gids = gids_buf_.data(); + + // A Categorical code and a Date are both 32 bits wide, so when both + // keys are one of those the composite fits in 64 bits exactly and can + // be probed in the same flat int map the single-int path uses. That is + // the common shape of `by { symbol, day }`, and it halves the key + // width, the hash and the stored entry against the 128-bit form. + // Both key domains are 32 bits wide here, so the composite is exact. + const auto pack_u64 = [](std::int64_t a, std::int64_t b) -> std::int64_t { + return static_cast( + (static_cast(static_cast(a)) << 32U) | + static_cast(static_cast(b))); + }; + if (pair_packs_u64_) { + // Both key domains are narrow enough to enumerate: a Categorical + // spans its dictionary, and a Date column's span is measured. When + // the product fits, index a flat cell -> gid array and the per-row + // hash disappears entirely -- the same trick, and the same reason, + // as the all-Categorical Cartesian path. `by { day }` over 4 + // distinct days was costing 34ms on 8M rows purely in hash probes. + if (try_process_rows_pair_dense(key_a_at, key_b_at, group_entries, agg_entries, rows)) { + return std::nullopt; + } + // Discovery across workers, for the case the dense array cannot + // hold: the cell budget is a product, so a wide symbol domain times + // a wide day domain overflows it long before either alone is + // remarkable, and the u64 key that falls out is the CHEAPEST key in + // this file to partition. Until now this branch returned before ever + // reaching `try_discover_partitioned` — a `by { symbol, day }` over + // 5000 symbols and 1000 days ran wholly serially. + // + // Only while the dense path has never run. Dense numbers groups in + // its own array and rebuilds that array from `pair_order_`, so it + // can safely take over from partitioned discovery; the reverse is + // not true, because the partitions would not know the groups dense + // had already numbered and would issue second ids for them. + if (!pair_dense_active_ && + try_discover_partitioned>( + [&](std::size_t row) { return pack_u64(key_a_at(row), key_b_at(row)); }, rows, + gids, int_partitions_, [&](std::size_t n) { pair_order_.resize(n); }, + [&](std::int64_t, std::uint32_t gid, std::size_t row) { + // From the row, not by unpacking: `pair_order_` holds the + // reader's own values, and the pack truncates to 32 bits. + pair_order_[gid] = {key_a_at(row), key_b_at(row)}; + }, + kDefaultPartitionMinRows, + [&](std::uint32_t gid) { + // The pack is a pure function of the pair, so a group's + // key is recoverable even though the pack is lossy. + return pack_u64(pair_order_[gid].first, pair_order_[gid].second); + })) { + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + // Falling here with groups already numbered means the dense path ran + // on an earlier chunk and this chunk's domains overflowed its budget. + // `int_index_` has no record of those groups, so without this it + // would issue a second id for each and the output would carry two + // rows per key. `pair_order_` is the pair path's source of truth — + // this is the same rebuild dense itself does when its bounds move. + if (int_index_.size() < pair_order_.size()) { + int_index_.reserve(pair_order_.size()); + for (std::size_t g = 0; g < pair_order_.size(); ++g) { + int_index_.emplace(pack_u64(pair_order_[g].first, pair_order_[g].second), + static_cast(g)); + } + } + + std::int64_t prev_packed = 0; + std::uint32_t prev_gid_u64 = std::numeric_limits::max(); + bool have_prev_u64 = false; + for (std::size_t row = 0; row < rows; ++row) { + const std::int64_t a = key_a_at(row); + const std::int64_t b = key_b_at(row); + const std::int64_t key = pack_u64(a, b); + std::uint32_t gid{}; + if (have_prev_u64 && key == prev_packed) { + gid = prev_gid_u64; + } else { + auto it = int_index_.find(key); + if (it == int_index_.end()) { + gid = static_cast(n_groups_); + int_index_.emplace(key, gid); + pair_order_.emplace_back(a, b); + ++n_groups_; + size_group_arrays(); + } else { + gid = it->second; + } + prev_packed = key; + prev_gid_u64 = gid; + have_prev_u64 = true; + } + gids[row] = gid; + } + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + if (try_owned( + [&](std::size_t row) { return pack(key_a_at(row), key_b_at(row)); }, rows, gids, + agg_entries, owned_pair_partitions_, kPairOwnedMinRows)) { + publish_fused_accumulation(); + return std::nullopt; + } + + if (try_discover_partitioned( + [&](std::size_t row) { return pack(key_a_at(row), key_b_at(row)); }, rows, gids, + pair_partitions_, [&](std::size_t n) { pair_order_.resize(n); }, + [&](const PairIntKey& key, std::uint32_t gid, std::size_t) { + pair_order_[gid] = {static_cast(key.first), + static_cast(key.second)}; + }, + kDefaultPartitionMinRows, + [&](std::uint32_t gid) { + return pack(pair_order_[gid].first, pair_order_[gid].second); + })) { + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + PairIntKey prev_key{}; + std::uint32_t prev_gid = std::numeric_limits::max(); + bool have_prev = false; + for (std::size_t row = 0; row < rows; ++row) { + const std::int64_t a = key_a_at(row); + const std::int64_t b = key_b_at(row); + const PairIntKey key = pack(a, b); + std::uint32_t gid{}; + if (have_prev && key == prev_key) { + gid = prev_gid; + } else { + auto it = pair_index_.find(key); + if (it == pair_index_.end()) { + gid = static_cast(n_groups_); + pair_index_.emplace(key, gid); + pair_order_.emplace_back(a, b); + ++n_groups_; + size_group_arrays(); + } else { + gid = it->second; + } + prev_key = key; + prev_gid = gid; + have_prev = true; + } + gids[row] = gid; + } + + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + /// Bounds of one key column over a chunk, as the dense cell numbering needs + /// them. A Categorical answers from its dictionary without reading a row; + /// anything else is measured. + template + auto key_bounds(const Reader& read, const ColumnValue& col, IntKeyKind kind, std::size_t rows) + -> std::pair { + if (kind == IntKeyKind::Cat) { + const auto size = std::get>(col).dictionary().size(); + return {0, size == 0 ? 0 : static_cast(size) - 1}; + } + std::int64_t lo = read(0); + std::int64_t hi = lo; + for (std::size_t row = 1; row < rows; ++row) { + const std::int64_t v = read(row); + lo = std::min(lo, v); + hi = std::max(hi, v); + } + return {lo, hi}; + } + + /// Group a packed 32-bit key pair through a flat cell array. Returns false + /// when the key domains are too large to enumerate, leaving the caller on + /// the hash path. + /// + /// The cell numbering is a function of the bounds, so a later chunk that + /// widens them invalidates every cell already handed out. That is handled + /// the way the multi-key Categorical path handles a stride change: widen to + /// the union and rebuild the array from `pair_order_`, which holds each + /// group's key pair. Group ids themselves never move. + template + auto try_process_rows_pair_dense(const ReaderA& key_a_at, const ReaderB& key_b_at, + // One caller + const std::vector& group_entries, + const std::vector& agg_entries, + std::size_t rows) -> bool { + if (rows == 0) { + return false; + } + const auto [a_lo, a_hi] = + key_bounds(key_a_at, *group_entries[0]->column, int_key_kind_, rows); + const auto [b_lo, b_hi] = + key_bounds(key_b_at, *group_entries[1]->column, int_key_kind_b_, rows); + + std::int64_t a_min = a_lo; + std::int64_t b_min = b_lo; + std::int64_t a_max = a_hi; + std::int64_t b_max = b_hi; + if (pair_dense_active_) { + a_min = std::min(a_min, pair_dense_a_min_); + b_min = std::min(b_min, pair_dense_b_min_); + a_max = std::max(a_max, pair_dense_a_max_); + b_max = std::max(b_max, pair_dense_b_max_); + } + + // Spans are computed in unsigned arithmetic so a domain that legitimately + // straddles zero cannot overflow the subtraction. + const auto a_span = static_cast(a_max - a_min) + 1; + const auto b_span = static_cast(b_max - b_min) + 1; + if (b_span != 0 && a_span > kDenseCellLimit / b_span) { + return false; // product overflows the dense budget + } + const std::uint64_t cells = a_span * b_span; + if (cells > kDenseCellLimit) { + return false; + } + + const bool bounds_changed = !pair_dense_active_ || a_min != pair_dense_a_min_ || + b_min != pair_dense_b_min_ || b_span != pair_dense_b_span_; + if (bounds_changed) { + pair_dense_gid_.assign(static_cast(cells), kNoGid); + for (std::size_t g = 0; g < pair_order_.size(); ++g) { + const auto cell = + (static_cast(pair_order_[g].first - a_min) * b_span) + + static_cast(pair_order_[g].second - b_min); + pair_dense_gid_[static_cast(cell)] = static_cast(g); + } + pair_dense_a_min_ = a_min; + pair_dense_b_min_ = b_min; + pair_dense_a_max_ = a_max; + pair_dense_b_max_ = b_max; + pair_dense_b_span_ = b_span; + pair_dense_active_ = true; + } else if (pair_dense_gid_.size() < cells) { + pair_dense_gid_.resize(static_cast(cells), kNoGid); + pair_dense_a_max_ = a_max; + } + + gids_buf_.resize(rows); + auto* gids = gids_buf_.data(); + std::uint32_t* dense = pair_dense_gid_.data(); + for (std::size_t row = 0; row < rows; ++row) { + const std::int64_t a = key_a_at(row); + const std::int64_t b = key_b_at(row); + const auto cell = (static_cast(a - a_min) * b_span) + + static_cast(b - b_min); + std::uint32_t gid = dense[cell]; + if (gid == kNoGid) { + gid = static_cast(n_groups_); + dense[cell] = gid; + pair_order_.emplace_back(a, b); + ++n_groups_; + size_group_arrays(); + dense = pair_dense_gid_.data(); + } + gids[row] = gid; + } + + publish_discovered(agg_entries, rows); + return true; + } + + /// Slot-indexed boxed value, grown on first use. `slot_index` is the same + /// `gid * n_aggs_ + agg_i` that indexes `flat_slots_`. + auto text_at(std::size_t slot_index) -> ScalarValue& { + if (text_store_.size() < flat_slots_.size()) { + text_store_.resize(flat_slots_.size()); + } + return text_store_[slot_index]; + } + + /// Scratch for one (group, aggregate). Only valid when that aggregate + /// declared scratch_doubles > 0. + [[nodiscard]] auto scratch_for(std::size_t gid, std::size_t agg_i) -> double* { + return scratch_.data() + (gid * scratch_stride_) + scratch_offset_[agg_i]; + } + + /// Size every per-group array to `n_groups_`. THE ONLY PLACE THAT RESIZES + /// THEM — the grouping fast paths used to call `flat_slots_.resize()` + /// directly, and adding a second per-group array (scratch) to just one of + /// those call sites left the others reading a null pointer. + /// One hash partition's share of the group index. Partitions are disjoint by + /// construction — a key's partition is a function of its hash — so a worker + /// owning a partition owns every row and every group in it, and needs no + /// lock and no merge against the others. + /// `Eq` is spelled out so a transparent hash/equal pair can be used: the + /// string path stores owning `std::string` keys but probes with + /// `std::string_view`, and only pays the copy on a genuinely new group — + /// exactly what the serial `str_index_` does. + template > + struct KeyPartition { + robin_hood::unordered_flat_map index; + /// This partition's groups, in the order they were first seen — which, + /// because rows are scattered in row order, is ascending by first row. + /// That is what makes the final ordering a merge of already-sorted + /// lists rather than a sort. + std::vector gids; + std::vector first_rows; + std::vector keys; + /// How many of `keys` have already been written back to the caller's + /// gid-indexed key vector; the rest were added by the current chunk. + std::size_t stored = 0; + }; + + /// Discover groups across workers by hash-partitioning the rows. + /// + /// **Group DISCOVERY is the serial half of a high-cardinality group-by, and + /// it is the half that could not be threaded before.** `accumulate_gids` + /// already fans out the summing, but only once gids exist, and its gate + /// declines exactly when groups are numerous — `morsels * n_groups > rows/4` + /// — because merging per-morsel partial tables then costs more than the scan + /// it saved. PDS-H q20 (543k groups) and q13 (150k) both land there and run + /// wholly serially. + /// + /// Partitioning removes the merge instead of paying it. Each key belongs to + /// exactly one partition, so per-partition tables never have to be + /// reconciled: there is no partial to combine, only a concatenation. It also + /// shrinks each table to a P-th of the rows, which is most of the win at + /// this cardinality — the serial probe is cache-miss bound on a table far + /// larger than L2. + /// + /// Returns false when the shape does not justify it and the caller should + /// run its own serial loop. + /// + /// **Ordering.** Ibex reports groups in first-occurrence order, and gids + /// here are handed out by an atomic, so gid order is a race. The order is + /// recovered from data instead: every group records the row it was first + /// seen at, and `build_output_chunk` walks the groups by that. Rows scatter + /// into partitions in row order, so each partition's list is already + /// ascending and the global order is a P-way merge, not a sort. + /// + /// **Determinism.** A group's rows all live in one partition and are visited + /// in row order, so each group's values accumulate in exactly the order the + /// serial path would use. The output is byte-identical, not merely + /// equivalent — including the float sums. + /// Passed as `key_of_group` by a caller that cannot reconstruct a + /// partition key from a group id, which is what decides whether this path + /// may start part-way through a stream. Only the packed path is in that + /// position: its key is built from a ROW and is not invertible. + struct NoGroupKeys {}; + + template , typename KeyAt, + typename ResizeKeys, typename StoreKey, typename KeyOfGroup = NoGroupKeys> + auto try_discover_partitioned(const KeyAt& key_at, std::size_t rows, std::uint32_t* gids, + std::vector>& partitions, + const ResizeKeys& resize_keys, const StoreKey& store_key, + std::size_t min_rows = kDefaultPartitionMinRows, + const KeyOfGroup& key_of_group = {}) -> bool { + // Below `min_rows` the partition and scatter passes cost more than the + // serial probe they replace. High cardinality is not checkable up front + // — it is what discovery is about to find out — so row count is the only + // gate available, and a low-cardinality run of this size still wins from + // the smaller per-partition tables. + // + // It is a parameter because the break-even is a property of the KEY, not + // of partitioning: the serial probe a packed key replaces is far more + // expensive per row than the one an int key replaces, so it pays off + // sooner. Callers that do not pass it keep the original threshold. + // + // Once this path HAS run, every later chunk must take it too, however + // small. The groups it discovered live in `partitions`, and the serial + // loops probe `int_index_` / `str_index_`, which this path never + // populates — so a small trailing chunk falling back would not find the + // existing groups and would allocate second ids for them. The row gate + // therefore only guards the first use. + // + // The gate counts every row this operator has been OFFERED, not the + // rows in this call. They were the same number while a source produced + // one chunk; once it produces six, a per-call gate sees a sixth of the + // input and declines on a query that plainly qualifies. PDS-H q20 is + // exactly that: 909k rows over 543k groups, which activated this path + // as one chunk and lost it entirely as six, taking the aggregate from + // 50ms to 79ms and the query +23%. The threshold itself is unchanged — + // lowering it is a measured dead end, because the break-even is set by + // group CARDINALITY and a low-cardinality run of this size loses. + constexpr bool can_seed = !std::is_same_v; + // The Discovery node's worker cap and fan-out permission come from + // the plan (src/runtime/PARALLELISM.md); the operator keeps only the two + // checks it alone can make -- is it nested, did this operator's input so + // far clear the floor. `decline != None` folds in `!exec_->can_fan_out()` + // (the plan resolves `SingleCore` from it). `min_rows` is still the + // operator's: it is the radix strategy's own admission gate, stricter + // than `try_owned`'s, and lives beside the constant it names. + if (exec_ == nullptr || on_worker_pool_thread()) { + return false; + } + if (par_.discovery.decline != physical::FanOutDecline::None || + par_.discovery.worker_cap < 2) { + return false; + } + if (!partitioned_active_) { + if (std::max(rows_offered_, rows) < min_rows) { + return false; + } + // Starting part-way through means groups already exist, and they + // live in the serial index this path neither reads nor writes. + // They have to be moved across (below) or they would be issued + // second ids; a caller that cannot hand back their keys cannot + // start late at all. + if (n_groups_ > 0 && !can_seed) { + return false; + } + } + auto& pool = process_worker_pool(); + const std::size_t workers = par_.discovery.worker_cap; + std::size_t part_count = 1; + while (part_count * 2 <= workers) { + part_count *= 2; // a power of two, so the partition is a mask + } + note_partition_fanout(); + const std::uint64_t part_mask = part_count - 1; + if (partitions.size() < part_count) { + partitions.resize(part_count); + } + + // Adopt the groups the serial path already discovered, so this path can + // start on any chunk rather than only the first. Each keeps its + // existing global id, and `stored` is set past them all: the ordering + // merge below only visits entries added by the current chunk, so their + // `first_rows` are never read and the ids handed out here continue + // after them — which is the same invariant that lets one partitioned + // chunk follow another. + if constexpr (can_seed) { + if (!partitioned_active_ && n_groups_ > 0) { + Hash hasher; + for (std::uint32_t gid = 0; gid < static_cast(n_groups_); ++gid) { + auto key = key_of_group(gid); + auto& partition = partitions[static_cast(hasher(key) & part_mask)]; + partition.index.emplace(Key(key), + static_cast(partition.gids.size())); + partition.gids.push_back(gid); + partition.first_rows.push_back(0); + partition.keys.emplace_back(key); + } + for (auto& partition : partitions) { + partition.stored = partition.gids.size(); + } + } + } + + // Pass 1: partition of every row, and a per-range histogram. Ranges are + // contiguous so that the scatter below keeps rows in row order within a + // partition, which is what the ordering argument above depends on. + const std::size_t ranges = workers; + const std::size_t grain = (rows + ranges - 1) / ranges; + part_of_row_.resize(rows); + std::vector counts(ranges * part_count, 0); + { + auto batch = pool.submit(ranges, [&](std::size_t r) { + const std::size_t begin = r * grain; + const std::size_t end = std::min(rows, begin + grain); + std::size_t* row_counts = counts.data() + (r * part_count); + Hash hasher; + for (std::size_t row = begin; row < end; ++row) { + const auto part = static_cast(hasher(key_at(row)) & part_mask); + part_of_row_[row] = part; + ++row_counts[part]; + } + }); + batch.wait(); + } + + // Exclusive prefix sum, partition-major then range-major, so each range + // writes its own slice of each partition without touching a shared + // cursor. + std::vector offsets(ranges * part_count, 0); + std::vector part_begin(part_count + 1, 0); + { + std::size_t running = 0; + for (std::size_t p = 0; p < part_count; ++p) { + part_begin[p] = running; + for (std::size_t r = 0; r < ranges; ++r) { + offsets[(r * part_count) + p] = running; + running += counts[(r * part_count) + p]; + } + } + part_begin[part_count] = running; + } + + // Pass 2: scatter row indices into their partition's slice. + scatter_rows_.resize(rows); + { + auto batch = pool.submit(ranges, [&](std::size_t r) { + const std::size_t begin = r * grain; + const std::size_t end = std::min(rows, begin + grain); + std::size_t* cursor = offsets.data() + (r * part_count); + for (std::size_t row = begin; row < end; ++row) { + scatter_rows_[cursor[part_of_row_[row]]++] = row; + } + }); + batch.wait(); + } + + // Pass 3: each worker owns whole partitions. Ids assigned here are + // partition-LOCAL — a plain counter, no atomic — because a global id + // cannot be handed out in first-occurrence order until every partition + // has been seen. `gids[row]` therefore holds a local id until pass 4. + const std::uint64_t row_base = rows_seen_; + { + std::atomic cursor{0}; + auto batch = pool.submit(std::min(workers, part_count), [&](std::size_t) { + for (std::size_t p = cursor.fetch_add(1, std::memory_order_relaxed); p < part_count; + p = cursor.fetch_add(1, std::memory_order_relaxed)) { + auto& partition = partitions[p]; + for (std::size_t i = part_begin[p]; i < part_begin[p + 1]; ++i) { + const std::size_t row = scatter_rows_[i]; + // `auto`, not `Key`: the probe type may be a view onto + // the key column (strings), and materializing an owning + // key per ROW rather than per GROUP is the whole cost + // this path exists to avoid. + const auto key = key_at(row); + auto it = partition.index.find(key); + std::uint32_t local{}; + if (it == partition.index.end()) { + local = static_cast(partition.gids.size()); + partition.index.emplace(Key(key), local); + partition.gids.push_back(0); // filled below, in order + partition.first_rows.push_back(row_base + row); + partition.keys.emplace_back(key); + } else { + local = it->second; + } + gids[row] = local; + } + } + }); + batch.wait(); + } + + // Number this chunk's new groups in first-occurrence order, so that gid + // order IS that order and nothing downstream has to compensate. + // + // Emitting in discovery order and permuting at the end was tried first + // and is a trap: the permutation turns the emit's sequential walk of the + // slot array into a random gather, which on q18's 1.5m-group aggregate + // cost more than the parallel discovery saved (+25% median). Paying one + // ordered pass here instead keeps every later pass sequential. + // + // Each partition's new entries are already ascending by first row, so + // this is a P-way merge. Groups carried over from earlier chunks keep + // their ids: a group first seen in a later chunk necessarily has a later + // first row, so appending after them preserves the global order. + // + // Rescanning every partition per group is O(groups x partitions), and + // replacing it with the textbook heap (replace-top, one sift per group, + // O(groups log partitions)) is a MEASURED DEAD END: q18's merge went + // 25.6ms -> 27.1ms and q20's 7.9ms -> 13.1ms, suite +0.15% over 12 + // interleaved rounds. P is `part_count`, a power of two capped by the + // worker count -- 8 here. Eight predictable compares over an array that + // never leaves L1 beat three sift levels of data-dependent branching and + // struct moves. A heap would need dozens of runs before it paid. + const std::size_t base = n_groups_; + { + std::vector cursors(part_count); + for (std::size_t p = 0; p < part_count; ++p) { + cursors[p] = partitions[p].stored; + } + auto next = static_cast(base); + while (true) { + std::size_t best = part_count; + std::uint64_t best_row = std::numeric_limits::max(); + for (std::size_t p = 0; p < part_count; ++p) { + if (cursors[p] >= partitions[p].first_rows.size()) { + continue; + } + if (partitions[p].first_rows[cursors[p]] < best_row) { + best_row = partitions[p].first_rows[cursors[p]]; + best = p; + } + } + if (best == part_count) { + break; + } + partitions[best].gids[cursors[best]] = next++; + ++cursors[best]; + } + n_groups_ = next; + } + size_group_arrays(); + // The caller's gid-indexed key vector has to cover the ids just handed + // out before any of them is written back. + resize_keys(n_groups_); + for (auto& partition : partitions) { + for (std::size_t i = partition.stored; i < partition.gids.size(); ++i) { + // The third argument is the group's first row WITHIN THIS + // CHUNK. Only entries from `stored` on are visited, and those + // are exactly the groups this call discovered, so their first + // row is always local and `row_base` recovers it. A packed key + // is not invertible on its own — the packed path uses this row + // to read the original column values back for the output. + store_key(partition.keys[i], partition.gids[i], + static_cast(partition.first_rows[i] - row_base)); + } + partition.stored = partition.gids.size(); + } + + // Pass 4: local id -> global gid. The partition a row belongs to is + // already recorded, so this is a lookup, not a re-probe. + { + auto batch = pool.submit(ranges, [&](std::size_t r) { + const std::size_t begin = r * grain; + const std::size_t end = std::min(rows, begin + grain); + for (std::size_t row = begin; row < end; ++row) { + gids[row] = partitions[part_of_row_[row]].gids[gids[row]]; + } + }); + batch.wait(); + } + rows_seen_ += rows; + partitioned_active_ = true; + return true; + } + + void size_group_arrays() { + auto tail = flat_slots_.grow_uninitialized(n_groups_ * n_aggs_); + if (!fill_slots_parallel(tail)) { + SlotArray::fill_default(tail); + } + if (scratch_stride_ != 0) { + scratch_.resize(n_groups_ * scratch_stride_, 0.0); + } + } + + /// Zero a freshly grown slot tail across workers. Returns false when the + /// tail is too small to be worth a batch — which is every call from + /// `alloc_group`, where the tail is one slot — and the caller fills it + /// serially. + /// + /// Worth threading at all only because the cost is page faults rather than + /// bytes: the kernel materializes a page per fault, and eight threads + /// faulting disjoint pages fault in parallel. Eight threads buy about 1.3x, + /// not 8x — the fault path serializes on the kernel's own locks — so this is + /// a small win, not a lever. Sizing it against the rest of q18's aggregate + /// (3M groups): 80ms parallel discovery probe, 44ms this fill, 26ms serial + /// first-occurrence merge, 12ms key-array growth, 10ms accumulate. It is the + /// largest SERIAL block, which is why it is threaded first, and it is still + /// only a sixth of the operator. + auto fill_slots_parallel(std::span tail) -> bool { + // Sized so the batch (a submit plus a join) stays small against the + // work. Below a few megabytes the serial memset is already + // bandwidth-bound and has no faults left to hide. + constexpr std::size_t kMinTailBytes = 4UL << 20; + if (tail.size() * sizeof(AggSlotCore) < kMinTailBytes) { + return false; + } + if (exec_ == nullptr || par_.accumulation.decline != physical::FanOutDecline::None || + on_worker_pool_thread()) { + return false; + } + auto& pool = process_worker_pool(); + const std::size_t threads = std::min(std::size_t{16}, par_.accumulation.worker_cap); + if (threads < 2) { + return false; + } + const std::size_t grain = (tail.size() + threads - 1) / threads; + auto batch = pool.submit(threads, [&](std::size_t t) { + const std::size_t begin = t * grain; + if (begin >= tail.size()) { + return; + } + const std::size_t end = std::min(tail.size(), begin + grain); + SlotArray::fill_default(tail.subspan(begin, end - begin)); + }); + batch.wait(); + return true; + } + + auto alloc_group() -> std::uint32_t { + auto gid = static_cast(n_groups_); + ++n_groups_; + size_group_arrays(); + return gid; + } + + /// Seed `group_order_`/`key_index_` (the generic path's state) with `n` + /// groups a fast path already discovered, in the same first-seen order + /// the fast path used -- so gid `i` here matches the gid `flat_slots_`/ + /// `scratch_` already hold data for at index `i`. `key_at` builds the + /// full `Key` (every fast path here stores raw values only, never a + /// null, so every migrated `Key` has an empty null mask). + /// + /// Hashing goes through `hash_key_value`, which is defined to agree with + /// `hash_key_row` on every value both can express -- the invariant this + /// whole migration rests on: a later chunk's row-based probe and a + /// migrated group's stored hash must land the same value in the same + /// slot. `KeyRowIndex::rehash` reproduces the exact open-address + /// placement `find_or_insert` would have made one row at a time, so a + /// batch reseed and an incremental build agree on where every group ends + /// up. + template + void seed_generic_index_from_keys(std::size_t n, const KeyAt& key_at) { + group_order_.reserve(group_order_.size() + n); + key_index_.hashes.reserve(key_index_.hashes.size() + n); + for (std::size_t i = 0; i < n; ++i) { + Key key = key_at(i); + key_index_.hashes.push_back(hash_key_value(key)); + group_order_.push_back(std::move(key)); + } + std::size_t capacity = 1024; + while (capacity * 7 < key_index_.hashes.size() * 10) { + capacity *= 2; + } + key_index_.rehash(capacity); + } + + /// A single `int_key_kind_`-typed raw value, as a `ScalarValue` matching + /// what `push_key_value` would have built for the equivalent column. + static auto scalar_of_int_key(IntKeyKind kind, std::int64_t raw) -> ScalarValue { + switch (kind) { + case IntKeyKind::Date: + return Date{.days = static_cast(raw)}; + case IntKeyKind::Ts: + return Timestamp{.nanos = raw}; + case IntKeyKind::Int64: + case IntKeyKind::Cat: + break; + } + return raw; + } + + /// Fold the single-int fast path's raw values (int64 / Date / Timestamp, + /// as `process_rows_int` stores them) into the generic grouping path when + /// a later chunk brings a validity bitmap the fast path cannot express. + /// The accumulated slots stay put -- only the key->gid lookup is rebuilt. + void migrate_int_fast_path_to_generic() { + seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { + Key key; + key.values.push_back(scalar_of_int_key(int_key_kind_, int_order_[i])); + return key; + }); + int_fast_path_ = false; + } + + /// Same migration as `migrate_int_fast_path_to_generic`, for the + /// single-string fast path's `str_order_`. + void migrate_str_fast_path_to_generic() { + seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { + Key key; + key.values.emplace_back(str_order_[i]); + return key; + }); + str_fast_path_ = false; + } + + /// Fold the categorical fast path -- single-key (`cat_order_`, code == + /// dictionary index) or multi-key (`multi_cat_codes_flat_`, `n_keys` + /// codes per group) -- into the generic grouping path. A code only means + /// something against ITS column's dictionary, which `group_templates_` + /// still holds (the empty `make_empty_like` template built at + /// `initialized_` time shares the dictionary every chunk's column uses), + /// so decoding a migrated group's code to the same string `push_key_value` + /// would have read off the live column is just a dictionary lookup. + void migrate_cat_fast_path_to_generic(std::size_t n_keys) { + const auto decode = [&](std::size_t c, Column::code_type code) { + return std::get>(group_templates_[c]) + .dictionary()[static_cast(code)]; + }; + if (n_keys == 1) { + seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { + Key key; + key.values.emplace_back(std::string(decode(0, cat_order_[i]))); + return key; + }); + } else { + seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { + Key key; + key.values.reserve(n_keys); + for (std::size_t c = 0; c < n_keys; ++c) { + key.values.emplace_back( + std::string(decode(c, multi_cat_codes_flat_[(i * n_keys) + c]))); + } + return key; + }); + } + cat_fast_path_ = false; + } + + /// Fold the paired-int fast path's raw values (`pair_order_`, one + /// `int64`/`Date`/`Timestamp`/categorical-code pair per group) into the + /// generic grouping path. `int_key_kind_`/`int_key_kind_b_` name each + /// column's type; a `Cat` column's code decodes through the matching + /// `group_templates_` entry exactly as the single/multi categorical + /// migration does. + void migrate_pair_int_fast_path_to_generic() { + const auto scalar_at = [&](IntKeyKind kind, std::size_t col_index, + std::int64_t raw) -> ScalarValue { + if (kind == IntKeyKind::Cat) { + const auto& dict = + std::get>(group_templates_[col_index]).dictionary(); + return std::string(dict[static_cast(raw)]); + } + return scalar_of_int_key(kind, raw); + }; + seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { + Key key; + key.values.reserve(2); + key.values.push_back(scalar_at(int_key_kind_, 0, pair_order_[i].first)); + key.values.push_back(scalar_at(int_key_kind_b_, 1, pair_order_[i].second)); + return key; + }); + pair_int_fast_path_ = false; + } + + /// Fold the packed (3+ key) fast path into the generic grouping path. + /// + /// Unlike every other fast path here, this one needs no decode at all: + /// `process_rows_packed` already builds a full boxed `Key` per group into + /// `group_order_` from the ROW (see its comment -- the packed word is + /// used only for the FAST lookup, never as the group's stored identity), + /// so `group_order_` is already exactly what the generic path expects. + /// Only `key_index_`, which hashes packed words rather than `Key`s, needs + /// rebuilding from what is already there. + void migrate_packed_fast_path_to_generic() { + key_index_.hashes.clear(); + key_index_.hashes.reserve(group_order_.size()); + for (const auto& key : group_order_) { + key_index_.hashes.push_back(hash_key_value(key)); + } + std::size_t capacity = 1024; + while (capacity * 7 < key_index_.hashes.size() * 10) { + capacity *= 2; + } + key_index_.rehash(capacity); + packed_fast_path_ = false; + } + + // ── Multi-key categorical index, keyed on the code tuple ────────────────── + // + // Groups are identified by the codes themselves (stored in + // multi_cat_codes_flat_), verified on every hit. The Cartesian cell is only + // a usable identity while the stride product fits in 64 bits; this does not + // care, and it survives dictionary growth without a rebuild. + static auto hash_codes(const Column::code_type* codes, std::size_t n) + -> std::uint64_t { + std::uint64_t seed = 0; + for (std::size_t i = 0; i < n; ++i) { + key_hash_mix(seed, std::hash{}(static_cast(codes[i]))); + } + // Finalized for the same reason `hash_key_row` is: this index masks the + // LOW bits to pick a slot and probes linearly, and the combine above + // never diffuses into them. Categorical codes are small dense integers, + // which is precisely the input that makes the unfinalized combine a + // near-linear function of the key and turns the probe into one long + // cluster. Nothing outside this index consumes the value, so unlike the + // three in `interpreter_internal.hpp` it has no agreement to maintain. + return key_hash_finalize(seed); + } + + [[nodiscard]] auto codes_of_group(std::size_t group, std::size_t n_keys) const + -> const Column::code_type* { + return multi_cat_codes_flat_.data() + (group * n_keys); + } + + void multi_cat_rehash_slots(std::size_t capacity, std::size_t n_keys) { + multi_cat_slots_.assign(capacity, 0U); + const std::size_t mask = capacity - 1; + for (std::size_t group = 0; group < n_groups_; ++group) { + std::size_t probe = + static_cast(hash_codes(codes_of_group(group, n_keys), n_keys)) & mask; + while (multi_cat_slots_[probe] != 0) { + probe = (probe + 1) & mask; + } + multi_cat_slots_[probe] = static_cast(group) + 1; + } + } + + /// Rebuild the index from the groups already collected — used when the + /// dense array gives up, and to seed the table on first use. + void multi_cat_rehash_groups() { + const std::size_t n_keys = n_groups_ == 0 ? 0 : multi_cat_codes_flat_.size() / n_groups_; + std::size_t capacity = 1024; + while ((n_groups_ * 10) > (capacity * 7)) { + capacity *= 2; + } + multi_cat_rehash_slots(capacity, n_keys); + } + + template + auto multi_cat_find_or_insert(const Column::code_type* codes, std::size_t n_keys, + NewGroup&& new_group) -> std::uint32_t { + const std::uint64_t hash = hash_codes(codes, n_keys); + const std::size_t mask = multi_cat_slots_.size() - 1; + std::size_t probe = static_cast(hash) & mask; + while (true) { + const std::uint32_t slot = multi_cat_slots_[probe]; + if (slot == 0) { + const std::uint32_t gid = std::forward(new_group)(); + multi_cat_slots_[probe] = gid + 1; + if ((n_groups_ * 10) > (multi_cat_slots_.size() * 7)) { + multi_cat_rehash_slots(multi_cat_slots_.size() * 2, n_keys); + } + return gid; + } + const std::uint32_t gid = slot - 1; + if (std::equal(codes, codes + n_keys, codes_of_group(gid, n_keys))) { + return gid; + } + probe = (probe + 1) & mask; + } + } + + auto process_rows_cat(const std::vector& group_entries, + const std::vector& agg_entries, std::size_t rows) + -> std::optional { + std::vector*> cat_cols; + cat_cols.reserve(group_entries.size()); + for (const auto* e : group_entries) { + cat_cols.push_back(&std::get>(*e->column)); + } + const std::size_t n_keys = cat_cols.size(); + const bool single_key = n_keys == 1; + + if (single_key && rows > 0 && + try_process_rows_cat_parallel(*cat_cols[0], agg_entries, rows)) { + publish_fused_accumulation(); + return std::nullopt; + } + + gids_buf_.resize(rows); + auto* gids = gids_buf_.data(); + if (single_key) { + // A Categorical code is already a dense index into [0, dict_size), + // so map code → gid with a direct array instead of hashing. Dicts + // only grow and never reorder across chunks, so existing gids stay + // valid and new dict entries just extend the array with sentinels. + const auto* codes = cat_cols[0]->codes_data(); + const std::size_t dict_size = cat_cols[0]->dictionary().size(); + if (cat_dense_gid_.size() < dict_size) { + cat_dense_gid_.resize(dict_size, kNoGid); + } + std::uint32_t* dense = cat_dense_gid_.data(); + for (std::size_t row = 0; row < rows; ++row) { + const auto code = codes[row]; + std::uint32_t gid = dense[code]; + if (gid == kNoGid) { + gid = alloc_group(); + dense[code] = gid; + cat_order_.push_back(code); + } + gids[row] = gid; + } + } else { + // Multi-key: encode each row as a uint64_t Cartesian cell. + // Strides may grow across chunks if a chunk introduces new dict + // entries; we recompute per chunk and rebuild the index when that + // happens (rare — Categorical dicts are usually stable). + std::vector dict_sizes(n_keys); + for (std::size_t c = 0; c < n_keys; ++c) { + dict_sizes[c] = static_cast(cat_cols[c]->dictionary().size()); + if (dict_sizes[c] == 0) + dict_sizes[c] = 1; // avoid stride collapse + } + // Strides: cell = c0*s0 + c1*s1 + … with s_{n-1} = 1. + // + // The cell only identifies a key tuple while the stride product + // fits in 64 bits. Past that the multiply wraps, distinct tuples + // collide, and `total_cells` itself wraps — a product of exactly + // 2^64 (16 keys of 16 values, say) lands on 0, which would pass the + // dense-array bound and index a zero-length array. Detect the + // overflow and let the hash path, which identifies groups by their + // codes rather than by a cell, take over. + std::vector strides(n_keys); + std::uint64_t total_cells = 1; + bool cells_overflow = false; + { + std::uint64_t s = 1; + for (int ci = static_cast(n_keys) - 1; ci >= 0; --ci) { + strides[static_cast(ci)] = s; + const std::uint64_t size = dict_sizes[static_cast(ci)]; + if (s > std::numeric_limits::max() / size) { + cells_overflow = true; + break; + } + s *= size; + } + total_cells = s; + } + const bool dense_possible = !cells_overflow && total_cells <= kDenseCellLimit; + + // Hoist raw code pointers out of the row loop. + std::vector::code_type*> raws(n_keys); + for (std::size_t c = 0; c < n_keys; ++c) + raws[c] = cat_cols[c]->codes_data(); + + const auto cell_of_group = [&](std::size_t g) -> std::uint64_t { + std::uint64_t cell = 0; + for (std::size_t c = 0; c < n_keys; ++c) { + cell += static_cast(multi_cat_codes_flat_[(g * n_keys) + c]) * + strides[c]; + } + return cell; + }; + const auto new_group = [&](std::size_t row) -> std::uint32_t { + for (std::size_t c = 0; c < n_keys; ++c) + multi_cat_codes_flat_.push_back(raws[c][row]); + return alloc_group(); + }; + + // When the Cartesian cell space is bounded, index a dense array + // (one load per row, no hashing). If a later chunk grows the dicts + // past the limit — or past what 64 bits can encode — migrate the + // existing groups into the hash index once and stay there; dicts + // only grow, so the cell space never shrinks back. + if (multi_dense_ && !dense_possible) { + multi_cat_rehash_groups(); + std::vector().swap(multi_cat_cell_dense_); + multi_dense_ = false; + } + + if (multi_dense_) { + // Rebuild the dense array when strides change (new dict entries). + if (multi_cat_strides_ != strides) { + multi_cat_cell_dense_.assign(static_cast(total_cells), kNoGid); + for (std::size_t g = 0; g < n_groups_; ++g) + multi_cat_cell_dense_[cell_of_group(g)] = static_cast(g); + multi_cat_strides_ = strides; + } + std::uint32_t* dense = multi_cat_cell_dense_.data(); + if (n_keys == 2) { + const auto* k0 = raws[0]; + const auto* k1 = raws[1]; + const std::uint64_t s0 = strides[0]; + const std::uint64_t s1 = strides[1]; + for (std::size_t row = 0; row < rows; ++row) { + const std::uint64_t cell = (static_cast(k0[row]) * s0) + + (static_cast(k1[row]) * s1); + std::uint32_t gid = dense[cell]; + if (gid == kNoGid) { + gid = new_group(row); + dense[cell] = gid; + } + gids[row] = gid; + } + } else { + for (std::size_t row = 0; row < rows; ++row) { + std::uint64_t cell = 0; + for (std::size_t c = 0; c < n_keys; ++c) + cell += static_cast(raws[c][row]) * strides[c]; + std::uint32_t gid = dense[cell]; + if (gid == kNoGid) { + gid = new_group(row); + dense[cell] = gid; + } + gids[row] = gid; + } + } + } else { + // Hash fallback for cell spaces that are unbounded, or that no + // longer fit in 64 bits. It identifies a group by its codes, not + // by a cell: correct however the strides behave, and it needs no + // rebuild when a new dictionary entry changes them. + if (multi_cat_slots_.empty()) { + multi_cat_rehash_groups(); + } + std::vector::code_type> row_codes(n_keys); + for (std::size_t row = 0; row < rows; ++row) { + for (std::size_t c = 0; c < n_keys; ++c) { + row_codes[c] = raws[c][row]; + } + gids[row] = multi_cat_find_or_insert(row_codes.data(), n_keys, + [&] { return new_group(row); }); + } + } + } + + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + /// Row count below which partitioned discovery is not worth its fan-out, + /// for the int/string keys it was originally tuned against. + static constexpr std::size_t kDefaultPartitionMinRows = 1U << 18U; + /// The packed key's break-even is lower: its serial probe hashes and + /// compares a multi-column key, so there is more per-row work to move onto + /// the workers than a single int key offers. + static constexpr std::size_t kPackedPartitionMinRows = 1U << 15U; + + /// Everything one packed width needs on the group-by side. + template + struct PackedGroups { + robin_hood::unordered_flat_map index; + std::vector> partitions; + }; + + /// Three or more fixed-width key columns, grouped through one packed key. + /// + /// **The output key store is deliberately unchanged.** `group_order_` still + /// holds one boxed `Key` per group, built once per GROUP, so + /// `build_output_chunk` needs no packed case: this path sets none of the + /// `*_fast_path_` flags and lands in the same branch the generic path uses. + /// That is also why the key is rebuilt from the ROW rather than unpacked — + /// a packed key is not invertible on its own, since a Categorical cell + /// holds an operator-global interned id rather than the column's own code. + template + auto process_rows_packed(const std::vector& group_entries, + const std::vector& agg_entries, + const std::vector& cols, std::size_t rows, + PackedGroups& state) -> std::optional { + gids_buf_.resize(rows); + auto* gids = gids_buf_.data(); + + // The one place a Key gets built: once per group, never per row. + const auto build_key_at = [&](std::size_t row) { + Key key; + key.values.reserve(group_entries.size()); + for (const auto* entry : group_entries) { + push_key_value(key, *entry, row); + } + return key; + }; + const auto key_at = [&](std::size_t row) { + return PackedKeyEncoder::pack_row(cols, row); + }; + + if (try_discover_partitioned( + key_at, rows, gids, state.partitions, + [&](std::size_t n) { group_order_.resize(n); }, + [&](const Packed&, std::uint32_t gid, std::size_t row) { + group_order_[gid] = build_key_at(row); + }, + kPackedPartitionMinRows)) { + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + // Run-length shortcut, as in the string and int paths: sorted or chunked + // input often repeats the key, so skip the map lookup when it matches + // the previous row. + Packed prev_key{}; + std::uint32_t prev_gid = std::numeric_limits::max(); + bool have_prev = false; + for (std::size_t row = 0; row < rows; ++row) { + const Packed key = key_at(row); + std::uint32_t gid{}; + if (have_prev && key == prev_key) { + gid = prev_gid; + } else { + auto it = state.index.find(key); + if (it == state.index.end()) { + group_order_.push_back(build_key_at(row)); + gid = alloc_group(); + state.index.emplace(key, gid); + } else { + gid = it->second; + } + prev_key = key; + prev_gid = gid; + have_prev = true; + } + gids[row] = gid; + } + + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + auto process_rows_generic(const std::vector& group_entries, + const std::vector& agg_entries, std::size_t rows) + -> std::optional { + std::vector cols; + cols.reserve(group_entries.size()); + for (const auto* entry : group_entries) { + auto col = make_key_col(*entry); + if (!col.has_value()) { + return "group-by: unsupported key column type"; + } + cols.push_back(*col); + } + + gids_buf_.resize(rows); + auto* gids = gids_buf_.data(); + for (std::size_t row = 0; row < rows; ++row) { + gids[row] = key_index_.find_or_insert(group_order_, cols, row, [&] { + // The one place a Key gets built: once per group, not per row. + Key key; + key.values.reserve(group_entries.size()); + for (const auto* entry : group_entries) { + push_key_value(key, *entry, row); + } + group_order_.push_back(std::move(key)); + return alloc_group(); + }); + } + + publish_discovered(agg_entries, rows); + return std::nullopt; + } + + /// Store non-null First aggregates at the row that creates each new group. + /// `seeded[a]` is set only when the input column has no validity bitmap, + /// so the discovery row is necessarily the first value under First's + /// null-skipping semantics. The later accumulation pass may skip exactly + /// these fields; nullable First and every Last still scan normally. + void seed_discovery_first(std::size_t first_gid, const std::vector& first_rows, + const std::vector& agg_entries, + const std::vector& seeded) { + bool has_text = false; + for (std::size_t a = 0; a < n_aggs_; ++a) { + has_text = has_text || (seeded[a] != 0U && plan_[a].kind == ExprType::String); + } + if (has_text && text_store_.size() < flat_slots_.size()) { + text_store_.resize(flat_slots_.size()); + } + const auto seed_one = [&](std::size_t gid, std::size_t row, std::size_t a) { + const auto& entry = *agg_entries[a]; + auto& slot = flat_slots_[(gid * n_aggs_) + a]; + if (plan_[a].kind == ExprType::Double) { + slot.double_value = std::get>(*entry.column)[row]; + } else if (plan_[a].kind == ExprType::Int) { + slot.int_value = std::get>(*entry.column)[row]; + } else { + std::string value; + if (plan_[a].categorical) { + value = std::string(std::get>(*entry.column)[row]); + } else { + value = std::string(std::get>(*entry.column)[row]); + } + text_store_[(gid * n_aggs_) + a] = std::move(value); + } + slot.mark_present(); + }; + + // Ceiling and permission from the plan; the `parallel_min_rows` floor on + // the first-occurrence count stays here (the shared knob, not a phase + // constant). + std::size_t threads = 1; + if (exec_ != nullptr && par_.final_ordering.decline == physical::FanOutDecline::None && + !on_worker_pool_thread() && first_rows.size() >= exec_->parallel_min_rows) { + threads = par_.final_ordering.worker_cap; + } + if (threads >= 2) { + note_finalize_fanout(); + const std::size_t grain = (first_rows.size() + threads - 1) / threads; + auto batch = process_worker_pool().submit(threads, [&](std::size_t worker) { + const std::size_t begin = worker * grain; + const std::size_t end = std::min(first_rows.size(), begin + grain); + for (std::size_t local = begin; local < end; ++local) { + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (seeded[a] != 0U) { + seed_one(first_gid + local, first_rows[local], a); + } + } + } + }); + batch.wait(); + if (exec_->parallel_stats != nullptr) { + exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); + } + return; + } + + // Field-major serial order keeps each source column's reads together. + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (seeded[a] == 0U) { + continue; + } + for (std::size_t local = 0; local < first_rows.size(); ++local) { + seed_one(first_gid + local, first_rows[local], a); + } + } + } + + /// Parallel scatter-accumulate of an already-assigned gid array — the + /// shared back half of every hash group-by fast path (string, int, + /// int-pair, generic). Returns false when the shape is not worth it and + /// the caller should accumulate serially into `flat_slots_`. + /// + /// **The gid pass stays serial on purpose.** It mutates the group index + /// and it is what defines group ORDER — Ibex reports groups in observed + /// first-occurrence order, so assigning gids concurrently would either + /// change the answer or need a reconciliation pass costing more than the + /// scan. Once every row carries a gid the rest is a pure scatter-reduce, + /// which is the part worth threading: for an 8-aggregate query like q01 it + /// is the dominant cost (24% of the whole query by profile), while the gid + /// probe is a single packed-integer lookup per row. + /// + /// Reproducibility, stated exactly, because the two halves get confused: + /// the partition is derived from row count and group count alone — both + /// properties of the DATA — and morsels merge in ascending order, so the + /// result does not depend on the machine, the thread count, or the + /// schedule (verified byte-identical across 2/3/5/8/16 threads). It DOES + /// differ from the serial path in the last ulp, because summing per morsel + /// and merging is not the same order as summing down the rows — q01's + /// sum_disc_price moves at the 11th significant digit. That is inherent to + /// any partitioned float reduction, it matches what + /// `try_process_rows_cat_parallel` has always done, and if anything the + /// partitioned sum is the more accurate of the two. + /// + /// This mirrors `try_process_rows_cat_parallel`, which can skip the gid + /// pass entirely because a Categorical code is already a dense index. + auto try_accumulate_parallel(const std::uint32_t* gids, + const std::vector& agg_entries, + std::size_t rows, const std::vector* skip = nullptr) + -> bool { + // Partition on the data alone -- not `exec_->can_fan_out()`, the thread + // budget, or whether this runs on a pool thread. Those choose who + // executes the morsels; the cut decides the arithmetic, and a cut that + // varied with the schedule would let the same query answer differently + // on two machines. Morsels run inline below when fan-out is not + // available. + if (n_groups_ == 0 || n_aggs_ == 0) { + return false; + } + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (skip != nullptr && (*skip)[a] != 0U) { + continue; + } + if (!agg_is_combinable(plan_[a].func)) { + return false; + } + // A boxed First/Last value lives outside the slot array, so a + // private copy would not capture it. + if (plan_[a].kind != ExprType::Int && plan_[a].kind != ExprType::Double) { + return false; + } + } + + // Same budget and morsel shape as the Categorical path, for the same + // reasons: partial state is bounded by GROUP COUNT, and the merge costs + // one agg_combine per (morsel, group) while the scan it replaces costs + // one update per row. Fanning out only pays when the merge stays small + // against the scan — a high-cardinality group-by would merge more slots + // than it saved row updates. + constexpr std::size_t kMinRowsPerMorsel = 65536; + constexpr std::size_t kMaxMorsels = 64; + constexpr std::size_t kPartialBudgetBytes = 32UL << 20; + constexpr std::size_t kMergeToScanRatio = 4; + const std::size_t per_morsel_bytes = + n_groups_ * ((n_aggs_ * sizeof(AggSlotCore)) + (scratch_stride_ * sizeof(double))); + if (per_morsel_bytes == 0 || per_morsel_bytes > kPartialBudgetBytes) { + return false; + } + std::size_t morsels = std::clamp(rows / kMinRowsPerMorsel, 1, kMaxMorsels); + morsels = std::min(morsels, kPartialBudgetBytes / per_morsel_bytes); + if (morsels < 2 || morsels * n_groups_ > rows / kMergeToScanRatio) { + return false; + } + + const std::size_t stride = n_groups_ * n_aggs_; + const std::size_t scratch_span = n_groups_ * scratch_stride_; + const std::size_t grain = (rows + morsels - 1) / morsels; + std::vector partials(morsels * stride); + std::vector partial_scratch(morsels * scratch_span, 0.0); + + const auto run_morsel = [&](std::size_t m) { + const std::size_t begin = m * grain; + const std::size_t end = std::min(rows, begin + grain); + if (begin < end) { + accumulate_columns_into(gids, agg_entries, begin, end, &partials[m * stride], + partial_scratch.data() + (m * scratch_span), skip); + } + }; + const std::size_t threads = + exec_ != nullptr && par_.accumulation.decline == physical::FanOutDecline::None + ? std::min(morsels, par_.accumulation.worker_cap) + : std::size_t{1}; + const bool fanned_out = threads >= 2 && !on_worker_pool_thread(); + if (fanned_out) { + auto& pool = process_worker_pool(); + std::atomic cursor{0}; + auto batch = pool.submit(threads, [&](std::size_t) { + while (true) { + const std::size_t m = cursor.fetch_add(1, std::memory_order_relaxed); + if (m >= morsels) { + return; + } + run_morsel(m); + } + }); + batch.wait(); + } else { + for (std::size_t m = 0; m < morsels; ++m) { + run_morsel(m); + } + } + + for (std::size_t m = 0; m < morsels; ++m) { + const AggSlotCore* src = &partials[m * stride]; + const double* src_scratch = partial_scratch.data() + (m * scratch_span); + for (std::size_t g = 0; g < n_groups_; ++g) { + AggSlotCore* dst = &flat_slots_[g * n_aggs_]; + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (skip != nullptr && (*skip)[a] != 0U) { + continue; + } + const std::size_t off = (g * scratch_stride_) + scratch_offset_[a]; + agg_combine(dst[a], src[(g * n_aggs_) + a], plan_[a].func, plan_[a].kind, + scratch_stride_ == 0 ? nullptr : scratch_.data() + off, + scratch_stride_ == 0 ? nullptr : src_scratch + off); + } + } + } + if (fanned_out && exec_ != nullptr && exec_->parallel_stats != nullptr) { + // Counts a fan-out, not a partition. The morsels are cut the same + // way either way, so counting them when they ran inline would + // report parallelism that never happened. + exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); + } + return true; + } + + /// Accumulate `gids` either across workers or, when that is not worth it, + /// serially — the one call every gid-assigning fast path ends with. + void accumulate_gids(const std::uint32_t* gids, + const std::vector& agg_entries, std::size_t rows, + const std::vector* skip = nullptr) { + if (!try_accumulate_parallel(gids, agg_entries, rows, skip)) { + accumulate_columns_into(gids, agg_entries, 0, rows, flat_slots_.data(), scratch_.data(), + skip); + } + } + + /// Scatter-accumulate rows [begin, end) into `base`, indexed by + /// `gids[row] * n_aggs_ + agg_i`. `base` is the caller's slot array — + /// `flat_slots_` for the serial path, a worker-private array for the + /// parallel one — and `GidT` covers both assigned gids (uint32_t) and raw + /// Categorical codes (int32_t), which are already dense indices. + /// + template + void accumulate_columns_into(const GidT* gids, + const std::vector& agg_entries, + std::size_t begin, std::size_t end, AggSlotCore* base, + double* scratch_base, + const std::vector* skip = nullptr) { + AggSlotCore* fs = base; + const std::size_t rows = end; + for (std::size_t agg_i = 0; agg_i < n_aggs_; ++agg_i) { + if (skip != nullptr && (*skip)[agg_i] != 0U) { + continue; + } + // Takes GidT so a signed Categorical code indexes without an + // implicit narrowing conversion at each of the ~19 call sites. + const auto slot_for = [&](GidT g) -> AggSlotCore& { + return fs[(static_cast(g) * n_aggs_) + agg_i]; + }; + // Moment accumulators, laid out beside `base` and indexed the same + // way. A worker-private slot array needs a worker-private scratch + // to match, or two morsels would accumulate one group's variance + // into the same doubles. + const auto scratch_at = [&](GidT g) -> double* { + return scratch_base + (static_cast(g) * scratch_stride_) + + scratch_offset_[agg_i]; + }; + + if (plan_[agg_i].func == ir::AggFunc::Count) { + for (std::size_t row = begin; row < rows; ++row) { + slot_for(gids[row]).count++; + } + continue; + } + + const auto& entry = *agg_entries[agg_i]; + const ValidityBitmap* validity = + entry.validity.has_value() ? &*entry.validity : nullptr; + const bool has_nulls = validity != nullptr; + + if (plan_[agg_i].kind == ExprType::Double) { + const double* data = std::get>(*entry.column).data(); + switch (plan_[agg_i].func) { + case ir::AggFunc::Sum: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + slot.double_value += data[row]; + slot.mark_present(); + } + break; + case ir::AggFunc::Mean: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + slot.double_value += data[row]; + slot.count++; + } + break; + case ir::AggFunc::Min: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + const double v = data[row]; + slot.double_value = slot.present() ? std::min(slot.double_value, v) : v; + slot.mark_present(); + } + break; + case ir::AggFunc::Max: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + const double v = data[row]; + slot.double_value = slot.present() ? std::max(slot.double_value, v) : v; + slot.mark_present(); + } + break; + case ir::AggFunc::Stddev: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + agg_update_stddev(slot_for(gids[row]), scratch_at(gids[row])[0], + data[row]); + } + break; + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + double* scr = scratch_at(gids[row]); + agg_update_moments(slot_for(gids[row]), scr[0], scr[1], scr[2], + data[row]); + } + break; + case ir::AggFunc::First: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + if (!slot.present()) { + slot.double_value = data[row]; + slot.mark_present(); + } + } + break; + case ir::AggFunc::Last: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + slot.double_value = data[row]; + slot.mark_present(); + } + break; + default: + break; + } + } else if (plan_[agg_i].kind == ExprType::Int) { + const std::int64_t* data = std::get>(*entry.column).data(); + switch (plan_[agg_i].func) { + case ir::AggFunc::Sum: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + slot.int_value += data[row]; + slot.mark_present(); + } + break; + case ir::AggFunc::Mean: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + slot.double_value += static_cast(data[row]); + slot.count++; + } + break; + case ir::AggFunc::Min: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + const std::int64_t v = data[row]; + slot.int_value = slot.present() ? std::min(slot.int_value, v) : v; + slot.mark_present(); + } + break; + case ir::AggFunc::Max: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + const std::int64_t v = data[row]; + slot.int_value = slot.present() ? std::max(slot.int_value, v) : v; + slot.mark_present(); + } + break; + case ir::AggFunc::Stddev: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + agg_update_stddev(slot_for(gids[row]), scratch_at(gids[row])[0], + static_cast(data[row])); + } + break; + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + double* scr = scratch_at(gids[row]); + agg_update_moments(slot_for(gids[row]), scr[0], scr[1], scr[2], + static_cast(data[row])); + } + break; + case ir::AggFunc::First: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + if (!slot.present()) { + slot.int_value = data[row]; + slot.mark_present(); + } + } + break; + case ir::AggFunc::Last: + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + slot.int_value = data[row]; + slot.mark_present(); + } + break; + default: + break; + } + } else { + // ExprType::String — First/Last only (the type gate in + // process_chunk rejects every other function here). Covers + // both Column and Column; the two + // share ScalarValue{std::string} as the wire format via + // append_scalar, which pushes into a Categorical dictionary + // when the target column is Categorical. + const bool categorical = plan_[agg_i].categorical; + const auto value_at = [&](std::size_t row) -> std::string { + if (categorical) { + return std::string(std::get>(*entry.column)[row]); + } + return std::string(std::get>(*entry.column)[row]); + }; + if (plan_[agg_i].func == ir::AggFunc::First) { + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + if (!slot.present()) { + text_at((static_cast(gids[row]) * n_aggs_) + agg_i) = + value_at(row); + slot.mark_present(); + } + } + } else { + for (std::size_t row = begin; row < rows; ++row) { + if (has_nulls && !(*validity)[row]) + continue; + auto& slot = slot_for(gids[row]); + text_at((static_cast(gids[row]) * n_aggs_) + agg_i) = + value_at(row); + slot.mark_present(); + } + } + } + } + } + + /// Accumulate rows [begin, end) of a global aggregate into `slots` + /// (n_aggs_ entries, caller-owned). No gid indirection: the compiler sees + /// a plain reduction over a contiguous range, which is also what lets a + /// worker own a private copy. + /// Parallel single-key Categorical group-by. Returns false when the shape + /// is not eligible and the caller should run the serial path. + /// + /// A Categorical code is already a dense index into the dictionary, so a + /// worker needs no hash table at all: it accumulates into a private slot + /// array indexed by code. That bounds the partial state by DICTIONARY + /// SIZE, which is what makes per-worker group state affordable here and + /// keeps `by symbol` (a few hundred groups) cheap. + /// + /// Group order is Ibex's observed first-occurrence order, and the merge + /// preserves it exactly: morsels are contiguous ascending row ranges, and + /// merging them in ascending order while walking each morsel's own + /// first-seen code list visits codes in precisely the order a serial scan + /// would have met them. + auto try_process_rows_cat_parallel(const Column& cat, + const std::vector& agg_entries, + std::size_t rows) -> bool { + // Partition on the data alone -- not `exec_->can_fan_out()`, the thread + // budget, or whether this runs on a pool thread. Those choose who + // executes the morsels; the cut decides the arithmetic, and a cut that + // varied with the schedule would let the same query answer differently + // on two machines. Morsels run inline below when fan-out is not + // available. + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (!agg_is_combinable(plan_[a].func)) { + return false; + } + // As above: a boxed First/Last value lives outside the slot. + if (plan_[a].kind != ExprType::Int && plan_[a].kind != ExprType::Double) { + return false; + } + } + const std::size_t dict_size = cat.dictionary().size(); + if (dict_size == 0) { + return false; + } + + // Same row-derived partition as the global aggregate, then bounded by + // what the per-worker slot arrays cost. Both inputs (row count and + // dictionary size) are properties of the DATA, so the partition — and + // therefore the float reduction order — is still independent of the + // machine and the schedule. + constexpr std::size_t kMinRowsPerMorsel = 65536; + constexpr std::size_t kMaxMorsels = 64; + constexpr std::size_t kPartialBudgetBytes = 32UL << 20; + const std::size_t per_morsel_bytes = + dict_size * ((n_aggs_ * sizeof(AggSlotCore)) + (scratch_stride_ * sizeof(double))); + if (per_morsel_bytes == 0 || per_morsel_bytes > kPartialBudgetBytes) { + return false; // one worker's state alone blows the budget + } + std::size_t morsels = std::clamp(rows / kMinRowsPerMorsel, 1, kMaxMorsels); + morsels = std::min(morsels, kPartialBudgetBytes / per_morsel_bytes); + if (morsels < 2) { + return false; + } + // The merge costs one agg_combine per (morsel, dictionary entry), so it + // scales with GROUP COUNT while the scan it replaces scales with rows. + // Fanning out only pays when the merge stays small against the scan: + // `by symbol` (252 groups) merges ~4k slots against 1M rows, but + // `by user_id` (100k groups) would merge ~1M — more work than it saves, + // and measured as a 17% REGRESSION when a smaller slot let it through + // the memory gate. + constexpr std::size_t kMergeToScanRatio = 4; + if (morsels * dict_size > rows / kMergeToScanRatio) { + return false; + } + + const auto* codes = cat.codes_data(); + const std::size_t grain = (rows + morsels - 1) / morsels; + std::vector partials(morsels * dict_size * n_aggs_); + std::vector cat_partial_scratch(morsels * dict_size * scratch_stride_, 0.0); + // Per morsel, the codes it saw in first-occurrence order. + std::vector::code_type>> seen(morsels); + + const auto run_morsel = [&](std::size_t m, std::vector& local_seen) { + const std::size_t begin = m * grain; + const std::size_t end = std::min(rows, begin + grain); + if (begin >= end) { + return; + } + std::ranges::fill(local_seen, std::uint8_t{0}); + auto& order = seen[m]; + for (std::size_t row = begin; row < end; ++row) { + const auto code = codes[row]; + if (local_seen[static_cast(code)] == 0) { + local_seen[static_cast(code)] = 1; + order.push_back(code); + } + } + accumulate_columns_into(codes, agg_entries, begin, end, + &partials[m * dict_size * n_aggs_], + cat_partial_scratch.data() + (m * dict_size * scratch_stride_)); + }; + const std::size_t threads = + exec_ != nullptr && par_.accumulation.decline == physical::FanOutDecline::None + ? std::min(morsels, par_.accumulation.worker_cap) + : std::size_t{1}; + const bool fanned_out = threads >= 2 && !on_worker_pool_thread(); + if (fanned_out) { + auto& pool = process_worker_pool(); + std::atomic cursor{0}; + auto batch = pool.submit(threads, [&](std::size_t) { + std::vector local_seen(dict_size, 0); + while (true) { + const std::size_t m = cursor.fetch_add(1, std::memory_order_relaxed); + if (m >= morsels) { + return; + } + run_morsel(m, local_seen); + } + }); + batch.wait(); + } else { + std::vector local_seen(dict_size, 0); + for (std::size_t m = 0; m < morsels; ++m) { + run_morsel(m, local_seen); + } + } + + if (cat_dense_gid_.size() < dict_size) { + cat_dense_gid_.resize(dict_size, kNoGid); + } + for (std::size_t m = 0; m < morsels; ++m) { + const AggSlotCore* src = &partials[m * dict_size * n_aggs_]; + const double* src_scratch = + cat_partial_scratch.data() + (m * dict_size * scratch_stride_); + for (const auto code : seen[m]) { + const auto idx = static_cast(code); + std::uint32_t gid = cat_dense_gid_[idx]; + if (gid == kNoGid) { + gid = alloc_group(); + cat_dense_gid_[idx] = gid; + cat_order_.push_back(code); + } + AggSlotCore* dst = &flat_slots_[(static_cast(gid) * n_aggs_)]; + for (std::size_t a = 0; a < n_aggs_; ++a) { + agg_combine(dst[a], src[(idx * n_aggs_) + a], plan_[a].func, plan_[a].kind, + scratch_stride_ == 0 ? nullptr : scratch_for(gid, a), + scratch_stride_ == 0 + ? nullptr + : src_scratch + (idx * scratch_stride_) + scratch_offset_[a]); + } + } + } + if (fanned_out && exec_ != nullptr && exec_->parallel_stats != nullptr) { + // Counts a fan-out, not a partition. The morsels are cut the same + // way either way, so counting them when they ran inline would + // report parallelism that never happened. + exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); + } + return true; + } + + /// `scratch_base` is this caller's moment region for the single group — + /// `scratch_` serially, a worker-private slice per morsel in parallel. + void accumulate_ungrouped_range_impl(const std::vector& agg_entries, + std::size_t begin, std::size_t end, AggSlotCore* slots, + double* scratch_base) { + for (std::size_t agg_i = 0; agg_i < n_aggs_; ++agg_i) { + AggSlotCore& slot = slots[agg_i]; + const auto func = plan_[agg_i].func; + if (func == ir::AggFunc::Count) { + slot.count += static_cast(end - begin); + continue; + } + const auto& entry = *agg_entries[agg_i]; + const ValidityBitmap* validity = + entry.validity.has_value() ? &*entry.validity : nullptr; + const bool has_nulls = validity != nullptr; + + // One generic driver per storage kind; `step` is the per-row body. + const auto each = [&](auto&& step) { + if (has_nulls) { + for (std::size_t row = begin; row < end; ++row) { + if (!(*validity)[row]) { + continue; + } + step(row); + } + } else { + for (std::size_t row = begin; row < end; ++row) { + step(row); + } + } + }; + + if (plan_[agg_i].kind == ExprType::Double) { + const double* data = std::get>(*entry.column).data(); + switch (func) { + case ir::AggFunc::Sum: + // has_value must track "saw a non-null value", not + // "the range was non-empty": sum over an all-null + // column is NULL, not 0. + each([&](std::size_t r) { + slot.double_value += data[r]; + slot.mark_present(); + }); + break; + case ir::AggFunc::Mean: + each([&](std::size_t r) { + slot.double_value += data[r]; + slot.count++; + }); + break; + case ir::AggFunc::Min: + each([&](std::size_t r) { + slot.double_value = + slot.present() ? std::min(slot.double_value, data[r]) : data[r]; + slot.mark_present(); + }); + break; + case ir::AggFunc::Max: + each([&](std::size_t r) { + slot.double_value = + slot.present() ? std::max(slot.double_value, data[r]) : data[r]; + slot.mark_present(); + }); + break; + case ir::AggFunc::Stddev: + each([&](std::size_t r) { + agg_update_stddev(slot, scratch_base[scratch_offset_[agg_i]], data[r]); + }); + break; + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + each([&](std::size_t r) { + double* scr = scratch_base + scratch_offset_[agg_i]; + agg_update_moments(slot, scr[0], scr[1], scr[2], data[r]); + }); + break; + case ir::AggFunc::First: + each([&](std::size_t r) { + if (!slot.present()) { + slot.double_value = data[r]; + slot.mark_present(); + } + }); + break; + case ir::AggFunc::Last: + each([&](std::size_t r) { + slot.double_value = data[r]; + slot.mark_present(); + }); + break; + default: + break; + } + } else if (plan_[agg_i].kind == ExprType::Int) { + const std::int64_t* data = std::get>(*entry.column).data(); + switch (func) { + case ir::AggFunc::Sum: + each([&](std::size_t r) { + slot.int_value += data[r]; + slot.mark_present(); + }); + break; + case ir::AggFunc::Mean: + each([&](std::size_t r) { + slot.double_value += static_cast(data[r]); + slot.count++; + }); + break; + case ir::AggFunc::Min: + each([&](std::size_t r) { + slot.int_value = + slot.present() ? std::min(slot.int_value, data[r]) : data[r]; + slot.mark_present(); + }); + break; + case ir::AggFunc::Max: + each([&](std::size_t r) { + slot.int_value = + slot.present() ? std::max(slot.int_value, data[r]) : data[r]; + slot.mark_present(); + }); + break; + case ir::AggFunc::Stddev: + each([&](std::size_t r) { + agg_update_stddev(slot, scratch_base[scratch_offset_[agg_i]], + static_cast(data[r])); + }); + break; + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + each([&](std::size_t r) { + double* scr = scratch_base + scratch_offset_[agg_i]; + agg_update_moments(slot, scr[0], scr[1], scr[2], + static_cast(data[r])); + }); + break; + case ir::AggFunc::First: + each([&](std::size_t r) { + if (!slot.present()) { + slot.int_value = data[r]; + slot.mark_present(); + } + }); + break; + case ir::AggFunc::Last: + each([&](std::size_t r) { + slot.int_value = data[r]; + slot.mark_present(); + }); + break; + default: + break; + } + } else { + // ExprType::String — First/Last only, same wire format as the + // grouped path (ScalarValue{std::string}). + const bool categorical = plan_[agg_i].categorical; + const auto value_at = [&](std::size_t row) -> std::string { + if (categorical) { + return std::string(std::get>(*entry.column)[row]); + } + return std::string(std::get>(*entry.column)[row]); + }; + if (func == ir::AggFunc::First) { + each([&](std::size_t r) { + if (!slot.present()) { + text_at(agg_i) = value_at(r); + slot.mark_present(); + } + }); + } else { + each([&](std::size_t r) { + text_at(agg_i) = value_at(r); + slot.mark_present(); + }); + } + } + } + } + + /// Global aggregate over `rows`, optionally fanned out across workers. + auto process_rows_ungrouped(const std::vector& agg_entries, + std::size_t rows) -> std::optional { + // An empty input must produce NO group, hence no output row — the + // generic path got that for free by only creating a group when a row + // arrived. Creating it up front turned `count()` over an empty table + // into a 1-row answer. + if (rows == 0) { + return std::nullopt; + } + if (n_groups_ == 0) { + // build_output_chunk() reads group_order_[g] for the generic key + // layout, so the single group still needs its (empty) Key — the + // generic path used to push one from its make_group lambda. + group_order_.emplace_back(); + alloc_group(); + } + AggSlotCore* dst = flat_slots_.data(); + + const std::size_t morsels = ungrouped_morsels(rows); + if (morsels < 2) { + accumulate_ungrouped_range_impl(agg_entries, 0, rows, dst, scratch_.data()); + return std::nullopt; + } + + // One private slot array per morsel, each written by exactly one + // worker. Merging them in ascending morsel order — never completion + // order — is what keeps First/Last correct and the float reduction + // reproducible run to run. + const std::size_t grain = (rows + morsels - 1) / morsels; + std::vector partials(morsels * n_aggs_); + std::vector ung_scratch(morsels * scratch_stride_, 0.0); + + // One morsel's work, identical whoever runs it -- which is the point: + // the partial it writes and the slot it lands in depend on `m` alone. + const auto run_morsel = [&](std::size_t m) { + const std::size_t begin = m * grain; + const std::size_t end = std::min(rows, begin + grain); + if (begin < end) { + accumulate_ungrouped_range_impl(agg_entries, begin, end, &partials[(m * n_aggs_)], + ung_scratch.data() + (m * scratch_stride_)); + } + }; + const std::size_t threads = + exec_ != nullptr && par_.accumulation.decline == physical::FanOutDecline::None + ? std::min(morsels, par_.accumulation.worker_cap) + : std::size_t{1}; + // Submitting from a pool thread would deadlock (`WorkerPool::submit` + // aborts rather than allow it), and one worker gains nothing from a + // round trip, so both run the morsels here. The arithmetic is unchanged + // either way. + if (threads >= 2 && !on_worker_pool_thread()) { + auto& pool = process_worker_pool(); + std::atomic cursor{0}; + auto batch = pool.submit(threads, [&](std::size_t) { + while (true) { + const std::size_t m = cursor.fetch_add(1, std::memory_order_relaxed); + if (m >= morsels) { + return; + } + run_morsel(m); + } + }); + batch.wait(); + if (exec_ != nullptr && exec_->parallel_stats != nullptr) { + exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); + } + } else { + for (std::size_t m = 0; m < morsels; ++m) { + run_morsel(m); + } + } + for (std::size_t m = 0; m < morsels; ++m) { + for (std::size_t a = 0; a < n_aggs_; ++a) { + agg_combine(dst[a], partials[(m * n_aggs_) + a], plan_[a].func, plan_[a].kind, + scratch_stride_ == 0 ? nullptr : scratch_for(0, a), + scratch_stride_ == 0 + ? nullptr + : ung_scratch.data() + (m * scratch_stride_) + scratch_offset_[a]); + } + } + return std::nullopt; + } + + /// How many row-morsels to split a global aggregate into; 1 = stay serial. + [[nodiscard]] auto ungrouped_morsels(std::size_t rows) const -> std::size_t { + // Deliberately NOT gated on `exec_->can_fan_out()`, the thread budget, or + // whether this runs on a pool thread. Those decide who EXECUTES the + // morsels, not how the range is cut, and a float reduction's result + // depends on where it is cut. Keeping the cut a function of the data + // alone is what makes one worker, eight workers, a serial run and a + // nested run agree bit for bit. The caller runs the morsels inline + // when it cannot fan out. + for (std::size_t a = 0; a < n_aggs_; ++a) { + if (!agg_is_combinable(plan_[a].func)) { + return 1; // Skew/Kurtosis: no partial merge, stay serial. + } + // A non-numeric First/Last keeps its value in `text_store_`, which + // agg_combine cannot reach and workers must not write concurrently. + if (plan_[a].kind != ExprType::Int && plan_[a].kind != ExprType::Double) { + return 1; + } + } + // The partition is a function of the ROW COUNT ALONE — deliberately + // not of the thread count. A float reduction's result depends on where + // the range is cut, so deriving morsels from the pool size would make + // `sum`/`std` answers differ between a 4-core box and a 24-core one, + // and differ again under `--threads`. Keyed on rows, the answer depends + // only on the data: same input, same result, any machine, any schedule. + // + // Morsels are large because a reduction's per-row cost is constant — + // equal ranges finish together, so unlike a filter there is no + // imbalance to hedge against and every extra morsel is pure dispatch + // and merge overhead. The cap bounds the partial array. + constexpr std::size_t kMinRowsPerMorsel = 65536; + constexpr std::size_t kMaxMorsels = 64; + return std::clamp(rows / kMinRowsPerMorsel, 1, kMaxMorsels); + } + + auto build_output_chunk() -> std::expected, std::string> { + Chunk out; + out.columns.reserve(group_by_->size() + aggregations_->size()); + + if (!initialized_) { + // No input rows at all — emit a chunk with empty columns of + // the expected schema where possible. Without any chunk we + // have no types to build group columns; return an empty + // optional so the sink finalizes an empty table. + return std::optional{}; + } + + for (std::size_t i = 0; i < group_by_->size(); ++i) { + out.add_column((*group_by_)[i].name, make_empty_like(group_templates_[i])); + } + for (std::size_t i = 0; i < aggregations_->size(); ++i) { + const auto& agg = (*aggregations_)[i]; + ColumnValue column; + switch (agg.func) { + case ir::AggFunc::Count: + column = Column{}; + break; + case ir::AggFunc::Mean: + case ir::AggFunc::Stddev: + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + column = Column{}; + break; + case ir::AggFunc::Sum: + case ir::AggFunc::Min: + case ir::AggFunc::Max: + if (plan_[i].kind == ExprType::Double) { + column = Column{}; + } else { + column = Column{}; + } + break; + case ir::AggFunc::First: + case ir::AggFunc::Last: + if (plan_[i].kind == ExprType::Double) { + column = Column{}; + } else if (plan_[i].kind == ExprType::Int) { + column = Column{}; + } else if (plan_[i].categorical) { + column = Column{}; + } else { + column = Column{}; + } + break; + default: + return std::unexpected("HashAggregateState: unsupported agg in build_output"); + } + out.add_column(agg.alias, std::move(column)); + } + + for (std::size_t i = 0; i < out.columns.size(); ++i) { + std::visit([&](auto& c) { c.reserve(n_groups_); }, out.mutable_column(i)); + } + + std::vector agg_validity(aggregations_->size()); + std::vector track_validity(aggregations_->size(), 0U); + for (std::size_t i = 0; i < aggregations_->size(); ++i) { + if (chunked_agg_tracks_validity(plan_[i].func)) { + track_validity[i] = 1U; + agg_validity[i].reserve(n_groups_); + } + } + + // The null group's key cell carries the type's zero value plus a clear + // validity bit. Only the generic Key path can produce one — the cat/str + // fast paths above are only taken for key columns with no nulls. + std::vector key_validity(group_by_->size()); + std::uint64_t any_null_keys = 0; + if (!cat_fast_path_ && !str_fast_path_ && !int_fast_path_ && !pair_int_fast_path_) { + for (const auto& key : group_order_) { + any_null_keys |= key.null_mask; + } + if (any_null_keys != 0) { + for (auto& bitmap : key_validity) { + bitmap.assign(n_groups_, true); + } + } + } + + const auto push_int_key = [](ColumnValue& col, IntKeyKind kind, std::int64_t raw) { + switch (kind) { + case IntKeyKind::Int64: + std::get>(col).push_back(raw); + return; + case IntKeyKind::Date: + std::get>(col).push_back(Date{static_cast(raw)}); + return; + case IntKeyKind::Ts: + std::get>(col).push_back(Timestamp{raw}); + return; + case IntKeyKind::Cat: + // The output column is `make_empty_like` of the input, so it + // shares the input's dictionary and the stored code resolves + // against it. + std::get>(col).push_code( + static_cast::code_type>(raw)); + return; + } + }; + + const AggSlotCore* fs = flat_slots_.data(); + + // Emission is column-major, one output column per task: every column is + // a separate buffer written by exactly one worker, so no two tasks touch + // the same bytes and the emitted order is the group order regardless of + // which worker got which column. + const auto emit_key_column = [&](std::size_t ci) { + ColumnValue& col = out.mutable_column(ci); + if (cat_fast_path_) { + auto& cat_col = std::get>(col); + const std::size_t n_keys = group_by_->size(); + if (n_keys == 1) { + for (std::size_t g = 0; g < n_groups_; ++g) { + cat_col.push_code(cat_order_[g]); + } + } else { + for (std::size_t g = 0; g < n_groups_; ++g) { + cat_col.push_code(multi_cat_codes_flat_[(g * n_keys) + ci]); + } + } + } else if (str_fast_path_) { + auto& str_col = std::get>(col); + for (std::size_t g = 0; g < n_groups_; ++g) { + str_col.push_back(str_order_[g]); + } + } else if (int_fast_path_) { + for (std::size_t g = 0; g < n_groups_; ++g) { + push_int_key(col, int_key_kind_, int_order_[g]); + } + } else if (pair_int_fast_path_) { + const IntKeyKind kind = ci == 0 ? int_key_kind_ : int_key_kind_b_; + for (std::size_t g = 0; g < n_groups_; ++g) { + push_int_key(col, kind, ci == 0 ? pair_order_[g].first : pair_order_[g].second); + } + } else { + for (std::size_t g = 0; g < n_groups_; ++g) { + const Key& key = group_order_[g]; + if (ci >= key.values.size()) { + continue; + } + append_scalar(col, key.values[ci]); + if (any_null_keys != 0 && ci < kMaxKeyColumns && + (key.null_mask & (std::uint64_t{1} << ci)) != 0) { + key_validity[ci].set(g, false); + } + } + } + }; + + const auto emit_agg_column = [&](std::size_t i) { + ColumnValue& column = out.mutable_column(group_by_->size() + i); + const bool tracks_validity = track_validity[i] != 0U; + for (std::size_t g = 0; g < n_groups_; ++g) { + const AggSlotCore& slot = fs[(g * n_aggs_) + i]; + if (tracks_validity) { + agg_validity[i].push_back(chunked_agg_valid(plan_[i].func, slot)); + } + switch (plan_[i].func) { + case ir::AggFunc::Count: + append_scalar(column, slot.count); + break; + case ir::AggFunc::Mean: + append_scalar(column, + slot.count == 0 + ? 0.0 + : slot.double_value / static_cast(slot.count)); + break; + case ir::AggFunc::Sum: + case ir::AggFunc::Min: + case ir::AggFunc::Max: + if (plan_[i].kind == ExprType::Double) { + append_scalar(column, slot.double_value); + } else { + append_scalar(column, slot.int_value); + } + break; + case ir::AggFunc::Stddev: + append_scalar(column, agg_finalize_stddev(slot, scratch_for(g, i)[0])); + break; + case ir::AggFunc::Skew: + append_scalar(column, agg_finalize_skew(slot, scratch_for(g, i)[0], + scratch_for(g, i)[1])); + break; + case ir::AggFunc::Kurtosis: + append_scalar(column, agg_finalize_kurtosis(slot, scratch_for(g, i)[0], + scratch_for(g, i)[2])); + break; + case ir::AggFunc::First: + case ir::AggFunc::Last: + if (plan_[i].kind == ExprType::Double) { + append_scalar(column, slot.double_value); + } else if (plan_[i].kind == ExprType::Int) { + append_scalar(column, slot.int_value); + } else { + append_scalar(column, text_store_[(g * n_aggs_) + i]); + } + break; + default: + break; + } + } + }; + + const std::size_t n_out_columns = out.columns.size(); + const auto emit_column = [&](std::size_t c) { + if (c < group_by_->size()) { + emit_key_column(c); + } else { + emit_agg_column(c - group_by_->size()); + } + }; + + // A one-task budget would pay the pool round trip for work the calling + // thread is about to do anyway, so it stays serial. + auto& pool = process_worker_pool(); + const std::size_t threads = + exec_ != nullptr && par_.emission.decline == physical::FanOutDecline::None + ? std::min(n_out_columns, par_.emission.worker_cap) + : std::size_t{1}; + if (exec_ != nullptr && !on_worker_pool_thread() && threads > 1 && + n_groups_ >= par_.emission.row_floor) { + std::atomic cursor{0}; + auto batch = pool.submit(threads, [&](std::size_t) { + while (true) { + const std::size_t c = cursor.fetch_add(1, std::memory_order_relaxed); + if (c >= n_out_columns) { + return; + } + emit_column(c); + } + }); + batch.wait(); + if (exec_->parallel_stats != nullptr) { + exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); + } + } else { + for (std::size_t c = 0; c < n_out_columns; ++c) { + emit_column(c); + } + } + + for (std::size_t i = 0; i < aggregations_->size(); ++i) { + if (track_validity[i] == 0U || agg_validity[i].empty()) { + continue; + } + bool has_null = false; + for (std::size_t r = 0; r < agg_validity[i].size(); ++r) { + if (!agg_validity[i][r]) { + has_null = true; + break; + } + } + if (has_null) { + out.columns[group_by_->size() + i].validity = std::move(agg_validity[i]); + } + } + + for (std::size_t ci = 0; ci < group_by_->size() && ci < kMaxKeyColumns; ++ci) { + if ((any_null_keys & (std::uint64_t{1} << ci)) != 0) { + out.columns[ci].validity = std::move(key_validity[ci]); + } + } + + return std::optional{std::move(out)}; + } + + struct SlotPlan { + ir::AggFunc func = ir::AggFunc::Sum; + ExprType kind = ExprType::Int; + // Only meaningful when kind == String: disambiguates Column + // from Column for First/Last output construction, since + // expr_type_for_column collapses both to ExprType::String. + bool categorical = false; + /// Extra per-GROUP state this aggregate needs, in doubles. Zero for + /// almost everything, which is the point: state that only one + /// aggregate wants must not sit in AggSlotCore, where it would cost + /// every group of every query. Skew/Kurtosis declare 2 (the third and + /// fourth central moments); a future aggregate declares whatever it + /// needs without touching the slot. + std::uint32_t scratch_doubles = 0; + }; + + struct CatKey { + std::vector::code_type> codes; + auto operator==(const CatKey& o) const noexcept -> bool { return codes == o.codes; } + }; + + // Transparent hash/eq: lets `str_index_.find(string_view)` skip the + // allocation of a temporary std::string on every probe. + struct StrViewHash { + using is_transparent = void; + auto operator()(std::string_view s) const noexcept -> std::size_t { + return robin_hood::hash_bytes(s.data(), s.size()); + } + auto operator()(const std::string& s) const noexcept -> std::size_t { + return robin_hood::hash_bytes(s.data(), s.size()); + } + }; + struct StrViewEq { + using is_transparent = void; + auto operator()(const std::string& a, const std::string& b) const noexcept -> bool { + return a == b; + } + auto operator()(const std::string& a, std::string_view b) const noexcept -> bool { + return std::string_view(a) == b; + } + auto operator()(std::string_view a, const std::string& b) const noexcept -> bool { + return a == std::string_view(b); + } + }; + struct CatKeyHash { + auto operator()(const CatKey& k) const noexcept -> std::size_t { + std::size_t h = 0; + for (auto c : k.codes) { + h ^= robin_hood::hash::code_type>{}(c) + 0x9e3779b9 + + (h << 6U) + (h >> 2U); + } + return h; + } + }; + + OperatorPtr child_; + const std::vector* group_by_; + const std::vector* aggregations_; + const ExecutionContext* exec_; + std::optional columns_; + bool columns_bound_ = false; + bool input_consumed_ = false; + bool ordering_finalized_ = false; + bool emitted_ = false; + std::optional active_chunk_; + + bool initialized_ = false; + bool cat_fast_path_ = false; + bool str_fast_path_ = false; + std::size_t n_aggs_ = 0; + std::size_t n_groups_ = 0; + std::vector plan_; + std::vector group_templates_; + std::vector discovery_first_eligible_; + + /// Per-group scratch for aggregates that declared `scratch_doubles`, + /// laid out group-major: `scratch_[gid * scratch_stride_ + offset[agg]]`. + /// Group-major so several small consumers in one group share a cache line. + /// Stays EMPTY when no aggregate asks for any — the same "pay only if used" + /// rule as text_store_, and `double` keeps it trivially copyable so growth + /// is a memcpy. + std::vector scratch_; + std::size_t scratch_stride_ = 0; + std::vector scratch_offset_; + + /// Boxed First/Last values for non-numeric columns, parallel to + /// `flat_slots_` and indexed identically. Stays EMPTY — no allocation, no + /// ScalarValue construction — for an all-numeric query, which is why the + /// slot itself can be a POD. + std::vector text_store_; + + // Flat accumulator storage: n_groups_ × n_aggs_ contiguous AggSlotCores. + SlotArray flat_slots_; + + // Reusable per-chunk gids buffer to avoid repeated heap allocations. + std::vector gids_buf_; + AggregateDiscoveryTransfer discovery_transfer_; + + // Generic path (non-Categorical group keys). + KeyRowIndex key_index_; + std::vector group_order_; + + // Sentinel for "no group assigned yet" in the dense index arrays. + static constexpr std::uint32_t kNoGid = std::numeric_limits::max(); + // Cartesian cell-space size below which multi-key grouping uses a dense + // array (one load per row) instead of hashing. 4M cells = 16 MB of u32. + static constexpr std::uint64_t kDenseCellLimit = 4'000'000ULL; + + // Single-Categorical fast path: code → gid via direct array (codes are a + // dense [0, dict_size) index, so no hashing is needed). + using cat_code = Column::code_type; + std::vector cat_dense_gid_; + std::vector cat_order_; + + // Multi-Categorical fast path: cell-encoded. Dense array while the cell + // space stays under kDenseCellLimit; spills to the hash map otherwise. + bool multi_dense_ = true; + std::vector multi_cat_cell_dense_; + std::vector multi_cat_slots_; // open addressing on the code tuple: gid + 1 + std::vector::code_type> multi_cat_codes_flat_; // n_groups_ × n_keys + std::vector multi_cat_strides_; // last-seen strides for rebuild detection + + // Single-string-key fast path. + robin_hood::unordered_flat_map str_index_; + std::vector str_order_; + + // Single fixed-width-integer-key fast path (int64 / Date / Timestamp, no + // nulls): a direct value -> gid map, no owned Key per group. `group by ` is one of the most common shapes, and the generic path was building a + // heap-allocated Key per group for it (117k allocations on TPC-H q02's + // 117k-group min). + bool int_fast_path_ = false; + IntKeyKind int_key_kind_ = IntKeyKind::Int64; + robin_hood::unordered_flat_map int_index_; + std::vector int_order_; ///< group keys, as raw integers, in first-seen order + + // Two fixed-width-integer keys are packed into a two-word composite key + // and grouped exactly as one integer key: `(l_partkey, l_suppkey)` on + // TPC-H q20's ~800k groups otherwise falls to the generic `Key` path, which + // boxes a ScalarValue-vector Key per group and compares it field-by-field on + // every probe. Keeping two 64-bit values is injective with no knowledge of + // their domains, so this is always exact and portable to MSVC. + bool pair_int_fast_path_ = false; + /// Three or more key columns, all fixed-width and packable. Sets no other + /// fast-path flag, so the output path treats it as the generic key case. + bool packed_fast_path_ = false; + PackedKeyEncoder encoder_; + PackedGroups> packed64_; + PackedGroups> packed128_; + PackedGroups> packed256_; + IntKeyKind int_key_kind_b_ = IntKeyKind::Int64; + struct PairIntKey { + std::uint64_t first = 0; + std::uint64_t second = 0; + + [[nodiscard]] friend auto operator==(const PairIntKey&, const PairIntKey&) + -> bool = default; + }; + struct PairIntKeyHash { + auto operator()(const PairIntKey& key) const noexcept -> std::size_t { + std::uint64_t h = key.first * 0x9e3779b97f4a7c15ULL; + h ^= key.second + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + return static_cast(h); + } + }; + robin_hood::unordered_flat_map pair_index_; + std::vector> pair_order_; + /// Parallel group discovery (see `try_discover_partitioned`). `rows_seen_` + /// makes a group's first-row index global across chunks, which is what the + /// first-occurrence numbering is merged on. + std::vector> pair_partitions_; + std::vector>> int_partitions_; + std::vector> str_partitions_; + std::vector part_of_row_; + std::vector scatter_rows_; + std::uint64_t rows_seen_ = 0; + + // --- Partition-owned aggregation (plans/parallelism-overview.md "stream + // multi-key joins" successor, step 2): the PairIntKey path only, admitted + // by `try_owned_pair`'s gates. `IBEX_DISABLE_OWNED_PAIR_AGG=1` is its kill + // switch. + + /// A partition's group discovery AND its own final aggregate state -- + /// no global gid, no global `flat_slots_` entry, until + /// `finalize_owned_pair` walks every partition once at + /// final emission to restore first-occurrence order (plans doc: "moves + /// that merge... does not eliminate the ordered merge itself"). + template > + struct OwnedPartition { + robin_hood::unordered_flat_map index; + std::vector keys; + std::vector first_rows; + std::vector slots; ///< n_local_groups * n_aggs_ + }; + std::vector> owned_pair_partitions_; + std::vector>> owned_int_partitions_; + // q18's Polars-style sink: per-chunk 4096-slot hot reducers publish compact + // pre-aggregates asynchronously; one final owner task per cold partition + // builds the persistent maps. Jobs are declared before the task group so + // reverse destruction joins every capture before releasing its storage. + std::vector> owned_async_jobs_; + std::optional owned_async_group_; + std::optional owned_async_error_; + std::size_t owned_async_part_count_ = 0; + bool owned_async_hot_mode_ = false; + std::vector owned_ordered_run_keys_; + std::vector owned_ordered_run_counts_; + bool owned_ordered_run_mode_ = false; + bool owned_ordered_runs_nondecreasing_ = true; + /// Set once this operator has committed to owned-partition mode. Per the + /// plan's safety note, only ever ADMITTED before any other discovery path + /// (serial or `try_discover_partitioned`) has created a group -- widening + /// to seed/migrate an in-progress run is out of scope for this prototype. + bool owned_mode_ = false; + bool owned_finalized_ = false; + std::uint64_t owned_rows_seen_ = 0; + /// Set once `try_discover_partitioned` has run; see the gate there for why + /// a later chunk may then never fall back to the serial loop. + bool partitioned_active_ = false; + /// Input rows this operator has been offered across every chunk, which is + /// what the partition gate measures. Distinct from `rows_seen_`, which + /// counts only rows the partitioned path itself consumed and exists to give + /// group first-rows a global base. + std::size_t rows_offered_ = 0; + /// The four structural nodes' fan-out policies, resolved by + /// `build_physical_aggregate` (src/runtime/PARALLELISM.md). The operator + /// reads `par_..{decline, worker_cap}` for fan-out permission and the + /// worker ceiling; it keeps only what the plan cannot know -- nesting + /// (`on_worker_pool_thread`), the data-derived partition/run terms of each + /// cap, and the strategy-specific admission floors (`kPairOwnedMinRows`, the + /// ordered-run `1U << 16U`, `parallel_min_rows`). + physical::AggregateParallelism par_{}; + ExecutionProfileEntry* discovery_profile_ = nullptr; + ExecutionProfileEntry* accumulation_profile_ = nullptr; + ExecutionProfileEntry* final_ordering_profile_ = nullptr; + ExecutionProfileEntry* emission_profile_ = nullptr; + /// Both keys are 32 bits wide (Categorical code / Date), so the composite + /// packs into 64 bits and probes `int_index_` instead of `pair_index_`. + /// The two paths are mutually exclusive, so sharing that map is safe. + bool pair_packs_u64_ = false; + /// Flat cell -> gid for a packed pair whose domains are small enough to + /// enumerate. Cells are numbered from the mins below, so widening them + /// rebuilds the array. + std::vector pair_dense_gid_; + std::int64_t pair_dense_a_min_ = 0; + std::int64_t pair_dense_a_max_ = 0; + std::int64_t pair_dense_b_min_ = 0; + std::int64_t pair_dense_b_max_ = 0; + std::uint64_t pair_dense_b_span_ = 0; + bool pair_dense_active_ = false; +}; + +/// Serial executor for the hash fallback's typed structural chain. Discovery +/// and Accumulation exchange a bounded per-chunk transfer (or an explicit fused +/// marker); FinalOrdering and Emission are separate calls with enforceable +/// preconditions. Keeping this coordinator outside the state object prevents +/// output construction from silently triggering the ordering merge again. +class HashAggregatePhaseOperator final : public Operator { + public: + explicit HashAggregatePhaseOperator(std::unique_ptr state) + : state_(std::move(state)) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (emitted_) { + return std::optional{}; + } + while (true) { + auto discovered = state_->next_discovery(); + if (!discovered.has_value()) { + return std::unexpected(std::move(discovered.error())); + } + if (!*discovered) { + break; + } + if (auto accumulated = state_->accumulate_discovery(); !accumulated.has_value()) { + return std::unexpected(std::move(accumulated.error())); + } + } + if (auto error = state_->finalize_ordering()) { + return std::unexpected(std::move(*error)); + } + emitted_ = true; + return state_->emit_output(); + } + + private: + std::unique_ptr state_; + bool emitted_ = false; +}; + +auto make_hash_aggregate_operator(OperatorPtr child, const std::vector* group_by, + const std::vector* aggregations, + const ExecutionContext& exec, physical::AggregateParallelism par, + std::optional columns) + -> OperatorPtr { + auto state = std::make_unique(std::move(child), group_by, aggregations, + exec, par, std::move(columns)); + return std::make_unique(std::move(state)); +} + +/// Replays one buffered chunk ahead of the rest of a child stream. Used by +/// ChunkedSortedAggregateOperator to hand the already-pulled first chunk back +/// to a fallback operator without losing it. +class PrependChunkOperator final : public Operator { + public: + PrependChunkOperator(Chunk first, OperatorPtr rest) + : first_(std::move(first)), rest_(std::move(rest)) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (!emitted_first_) { + emitted_first_ = true; + return std::optional{std::move(first_)}; + } + return rest_->next(); + } + + private: + Chunk first_; + OperatorPtr rest_; + bool emitted_first_ = false; +}; + +/// Streaming aggregate for input already sorted on the group-by keys. +/// +/// When the child's chunks declare an `ordering` whose leading keys cover the +/// group_by columns, every group's rows are contiguous in the stream. We then +/// keep accumulators for only the *current* group, emit each group as soon as +/// its run ends, and produce output already sorted by the group keys. Peak +/// memory is O(one group + one output chunk) instead of O(all groups), and +/// there is no hashing — group changes are detected by a typed equality scan. +/// +/// Eligibility is decided from the first non-empty chunk. If the input is not +/// sorted on the group_by keys (no `ordering`, or it doesn't cover them, or a +/// group key is nullable), the operator transparently falls back to the +/// hash aggregate phase operator by replaying the already-pulled chunk +/// ahead of the remaining child. The supported agg subset matches +/// HashAggregateState (Count/Sum/Min/Max/Mean on numeric columns); +/// build_operator only routes that subset here. +class ChunkedSortedAggregateOperator final : public Operator { + public: + ChunkedSortedAggregateOperator( + OperatorPtr child, const std::vector* group_by, + const std::vector* aggregations, const ExecutionContext& exec, + physical::AggregateParallelism par = {}, + std::optional columns = std::nullopt) + : child_(std::move(child)), + group_by_(group_by), + aggregations_(aggregations), + exec_(&exec), + par_(par), + columns_(std::move(columns)) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (fallback_) { + return fallback_->next(); + } + if (!decided_) { + auto decided = decide_strategy(); + if (!decided.has_value()) { + return std::unexpected(std::move(decided.error())); + } + if (fallback_) { + return fallback_->next(); + } + } + return next_sorted(); + } + + private: + struct SlotPlan { + ir::AggFunc func = ir::AggFunc::Sum; + ExprType kind = ExprType::Int; + }; + + // Pull chunks until the first non-empty one, then choose sorted vs fallback. + auto decide_strategy() -> std::expected { + decided_ = true; + Chunk first; + bool have = false; + std::optional schema_only; + while (true) { + auto chunk_res = child_->next(); + if (!chunk_res.has_value()) { + return std::unexpected(std::move(chunk_res.error())); + } + if (!chunk_res.value().has_value()) { + break; // EOF before any rows + } + if (chunk_res.value()->rows() == 0) { + // Empty, but it still carries the input's columns and their + // types. Keep the first one in case no chunk ever has rows. + if (!schema_only.has_value() && !chunk_res.value()->columns.empty()) { + schema_only = std::move(*chunk_res.value()); + } + continue; + } + first = std::move(*chunk_res.value()); + have = true; + break; + } + if (!have) { + // Every row was filtered away upstream. Emitting nothing would emit + // no schema either, and the result would materialize with no columns + // at all — so a downstream join looking for its key, or a filter for + // the value it compares, would fail with "unknown column" on what is + // really just an empty input. The hash operator derives its output + // columns from the input's types, so hand it the empty chunk and let + // it produce a properly-shaped empty result. + if (schema_only.has_value()) { + fallback_ = + make_hash_aggregate_operator(std::make_unique( + std::move(*schema_only), std::move(child_)), + group_by_, aggregations_, *exec_, par_, columns_); + return {}; + } + done_ = true; + input_eof_ = true; + return {}; + } + if (auto err = bind_aggregate_columns(columns_, columns_bound_, *group_by_, *aggregations_, + first)) { + return std::unexpected(std::move(*err)); + } + if (!sorted_on_group_by(first) || needs_hash_fallback(first)) { + fallback_ = make_hash_aggregate_operator( + std::make_unique(std::move(first), std::move(child_)), + group_by_, aggregations_, *exec_, par_, columns_); + return {}; + } + if (auto err = init_plan(first)) { + return std::unexpected(*err); + } + if (auto err = consume(first)) { + return std::unexpected(*err); + } + return {}; + } + + // The input is grouped-contiguous iff the first |group_by| ordering keys + // are exactly the group_by columns (as a set; direction and intra-prefix + // order don't matter for contiguity). Nullable group keys fall back, since + // the streaming key compare ignores validity. + [[nodiscard]] auto sorted_on_group_by(const Chunk& chunk) const -> bool { + if (!columns_.has_value() || group_by_->empty()) { + return false; // unbound, or a global aggregate: let the hash path handle it + } + if (!chunk.ordering().has_value() || chunk.ordering()->size() < group_by_->size()) { + return false; + } + const auto& ordering = *chunk.ordering(); + for (std::size_t i = 0; i < group_by_->size(); ++i) { + bool in_group = false; + for (const auto& g : *group_by_) { + if (g.name == ordering[i].name) { + in_group = true; + break; + } + } + if (!in_group) { + return false; + } + } + return std::ranges::all_of(columns_->group_by, [&chunk](const std::size_t index) { + return !chunk.columns[index].validity.has_value(); + }); + } + + // Non-numeric First/Last (string/categorical) has no group-at-a-time + // implementation here — route it to the hash operator, which handles any + // type. Numeric First/Last streams natively (see accumulate_typed). + [[nodiscard]] auto needs_hash_fallback(const Chunk& first) const -> bool { + if (!columns_.has_value()) { + return true; // unbound: route to the hash operator, which rebinds + } + for (std::size_t i = 0; i < aggregations_->size(); ++i) { + const ir::AggSpec& agg = (*aggregations_)[i]; + const std::optional& input_idx = columns_->aggregate_inputs[i]; + if ((agg.func != ir::AggFunc::First && agg.func != ir::AggFunc::Last) || + !input_idx.has_value()) { + continue; + } + const ColumnEntry* entry = &first.columns[*input_idx]; + const ExprType kind = expr_type_for_column(*entry->column); + if (kind != ExprType::Int && kind != ExprType::Double) { + return true; + } + } + return false; + } + + auto init_plan(const Chunk& first) -> std::optional { + if (!columns_.has_value()) { + return "ChunkedSortedAggregateOperator: column mapping not bound"; + } + n_aggs_ = aggregations_->size(); + plan_.resize(n_aggs_); + for (std::size_t i = 0; i < n_aggs_; ++i) { + const auto& agg = (*aggregations_)[i]; + plan_[i].func = agg.func; + if (agg.func == ir::AggFunc::Count) { + plan_[i].kind = ExprType::Int; + continue; + } + const std::optional& input_idx = columns_->aggregate_inputs[i]; + if (!input_idx.has_value()) { + continue; + } + const ColumnEntry* entry = &first.columns[*input_idx]; + const ExprType kind = expr_type_for_column(*entry->column); + if (kind != ExprType::Int && kind != ExprType::Double) { + return "ChunkedSortedAggregateOperator: non-numeric aggregation not supported"; + } + plan_[i].kind = kind; + } + key_templates_.clear(); + key_templates_.reserve(group_by_->size()); + for (const std::size_t index : columns_->group_by) { + key_templates_.push_back(make_empty_like(*first.columns[index].column)); + } + track_validity_.assign(n_aggs_, 0U); + for (std::size_t i = 0; i < n_aggs_; ++i) { + track_validity_[i] = chunked_agg_tracks_validity(plan_[i].func) ? 1U : 0U; + } + // Capture the leading ordering keys so emitted chunks can advertise the + // group-sorted order they preserve (lets a downstream `order` skip work). + if (first.ordering().has_value()) { + out_ordering_.assign( + first.ordering()->begin(), + first.ordering()->begin() + static_cast(group_by_->size())); + } + cur_slots_.assign(n_aggs_, AggSlotCore{}); + cur_scratch_.assign(n_aggs_ * kMomentScratch, 0.0); + reset_output(); + return std::nullopt; + } + + void reset_output() { + out_columns_.clear(); + out_columns_.reserve(group_by_->size() + n_aggs_); + for (std::size_t i = 0; i < group_by_->size(); ++i) { + ColumnEntry entry; + entry.name = (*group_by_)[i].name; + entry.column = std::make_shared(make_empty_like(key_templates_[i])); + std::visit([&](auto& c) { c.reserve(kEmitThreshold); }, *entry.column); + out_columns_.push_back(std::move(entry)); + } + for (std::size_t i = 0; i < n_aggs_; ++i) { + ColumnValue column; + switch (plan_[i].func) { + case ir::AggFunc::Count: + column = Column{}; + break; + case ir::AggFunc::Mean: + case ir::AggFunc::Stddev: + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + column = Column{}; + break; + default: // Sum / Min / Max + column = plan_[i].kind == ExprType::Double + ? ColumnValue{Column{}} + : ColumnValue{Column{}}; + break; + } + std::visit([&](auto& c) { c.reserve(kEmitThreshold); }, column); + ColumnEntry entry; + entry.name = (*aggregations_)[i].alias; + entry.column = std::make_shared(std::move(column)); + out_columns_.push_back(std::move(entry)); + } + out_validity_.assign(n_aggs_, ValidityBitmap{}); + for (std::size_t i = 0; i < n_aggs_; ++i) { + if (track_validity_[i] != 0U) { + out_validity_[i].reserve(kEmitThreshold); + } + } + pending_rows_ = 0; + } + + // Drive input until we have a full output batch or hit EOF, then emit. + auto next_sorted() -> std::expected, std::string> { + if (done_) { + return std::optional{}; + } + while (!input_eof_ && pending_rows_ < kEmitThreshold) { + auto chunk_res = child_->next(); + if (!chunk_res.has_value()) { + return std::unexpected(std::move(chunk_res.error())); + } + if (!chunk_res.value().has_value()) { + input_eof_ = true; + break; + } + if (chunk_res.value()->rows() == 0) { + continue; + } + if (auto err = consume(*chunk_res.value())) { + return std::unexpected(*err); + } + } + if (input_eof_ && open_) { + close_group(); + open_ = false; + } + if (pending_rows_ == 0) { + done_ = true; + return std::optional{}; + } + Chunk out = take_pending(); + if (input_eof_) { + done_ = true; + } + return std::optional{std::move(out)}; + } + + // Fold one chunk into the streaming state. Rows are scanned as runs of + // equal group keys; each run is accumulated columnwise into the open + // group's slots, and a group-key change closes the open group. + auto consume(const Chunk& chunk) -> std::optional { + if (!columns_.has_value()) { + return "ChunkedSortedAggregateOperator: column mapping not bound"; + } + std::vector key_cols; + key_cols.reserve(group_by_->size()); + for (const std::size_t index : columns_->group_by) { + key_cols.push_back(chunk.columns[index].column.get()); + } + std::vector agg_entries(n_aggs_, nullptr); + for (std::size_t i = 0; i < n_aggs_; ++i) { + const std::optional& input_idx = columns_->aggregate_inputs[i]; + if (plan_[i].func == ir::AggFunc::Count || !input_idx.has_value()) { + continue; + } + const ColumnEntry* entry = &chunk.columns[*input_idx]; + if (expr_type_for_column(*entry->column) != plan_[i].kind) { + return "ChunkedSortedAggregateOperator: aggregate column type changed across " + "chunks"; + } + agg_entries[i] = entry; + } + + const std::size_t rows = chunk.rows(); + std::size_t r = 0; + while (r < rows) { + if (!open_) { + start_group(key_cols, r); + } else if (!row_matches_open(key_cols, r)) { + close_group(); + start_group(key_cols, r); + } + std::size_t e = r + 1; + while (e < rows && cells_equal(key_cols, r, e)) { + ++e; + } + accumulate_range(agg_entries, r, e); + r = e; + } + return std::nullopt; + } + + void start_group(const std::vector& key_cols, std::size_t row) { + open_key_.clear(); + open_key_.reserve(key_cols.size()); + for (const auto* col : key_cols) { + open_key_.push_back(scalar_from_column(*col, row)); + } + std::ranges::fill(cur_slots_, AggSlotCore{}); + std::ranges::fill(cur_scratch_, 0.0); + open_ = true; + } + + // Whether `row` continues the currently open group. Only called at run + // anchors (group boundaries and chunk starts), so the scalar build is + // paid per group, not per row. + [[nodiscard]] auto row_matches_open(const std::vector& key_cols, + std::size_t row) const -> bool { + for (std::size_t i = 0; i < key_cols.size(); ++i) { + if (scalar_from_column(*key_cols[i], row) != open_key_[i]) { + return false; + } + } + return true; + } + + static auto cell_equal(const ColumnValue& col, std::size_t a, std::size_t b) -> bool { + return std::visit( + [&](const auto& c) -> bool { + using ColT = std::decay_t; + if constexpr (std::is_same_v>) { + return c.code_at(a) == c.code_at(b); + } else if constexpr (std::is_same_v>) { + return c[a].days == c[b].days; + } else if constexpr (std::is_same_v>) { + return c[a].nanos == c[b].nanos; + } else { + return c[a] == c[b]; + } + }, + col); + } + + [[nodiscard]] static auto cells_equal(const std::vector& key_cols, + std::size_t a, std::size_t b) -> bool { + return std::ranges::all_of(key_cols, + [a, b](const auto* col) { return cell_equal(*col, a, b); }); + } + + // Accumulate the contiguous row range [start, end) — all one group — into + // the open group's slots, branch-hoisted per aggregation. + void accumulate_range(const std::vector& agg_entries, std::size_t start, + std::size_t end) { + for (std::size_t i = 0; i < n_aggs_; ++i) { + AggSlotCore& slot = cur_slots_[i]; + if (plan_[i].func == ir::AggFunc::Count) { + slot.count += static_cast(end - start); + continue; + } + const auto& entry = *agg_entries[i]; + const bool has_nulls = entry.validity.has_value(); + if (plan_[i].kind == ExprType::Double) { + const double* data = std::get>(*entry.column).data(); + accumulate_typed(slot, &cur_scratch_[i * kMomentScratch], plan_[i].func, data, + entry, has_nulls, start, end); + } else { + const std::int64_t* data = std::get>(*entry.column).data(); + accumulate_typed(slot, &cur_scratch_[i * kMomentScratch], plan_[i].func, data, + entry, has_nulls, start, end); + } + } + } + + template + /// `scratch` is this aggregate's per-group scratch (2 doubles for the + /// higher moments); it stays a parameter so this helper remains static and + /// has no reach into operator state. + static void accumulate_typed(AggSlotCore& slot, double* scratch, ir::AggFunc func, + const T* data, const ColumnEntry& entry, bool has_nulls, + std::size_t start, std::size_t end) { + const auto valid = [&](std::size_t row) { return !has_nulls || (*entry.validity)[row]; }; + switch (func) { + case ir::AggFunc::Sum: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row)) { + continue; + } + if constexpr (std::is_same_v) { + slot.double_value += data[row]; + } else { + slot.int_value += data[row]; + } + slot.mark_present(); + } + break; + case ir::AggFunc::Mean: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row)) { + continue; + } + slot.double_value += static_cast(data[row]); + slot.count++; + } + break; + case ir::AggFunc::Min: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row)) { + continue; + } + if constexpr (std::is_same_v) { + slot.double_value = + slot.present() ? std::min(slot.double_value, data[row]) : data[row]; + } else { + slot.int_value = + slot.present() ? std::min(slot.int_value, data[row]) : data[row]; + } + slot.mark_present(); + } + break; + case ir::AggFunc::Max: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row)) { + continue; + } + if constexpr (std::is_same_v) { + slot.double_value = + slot.present() ? std::max(slot.double_value, data[row]) : data[row]; + } else { + slot.int_value = + slot.present() ? std::max(slot.int_value, data[row]) : data[row]; + } + slot.mark_present(); + } + break; + case ir::AggFunc::Stddev: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row)) { + continue; + } + agg_update_stddev(slot, scratch[0], static_cast(data[row])); + } + break; + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row)) { + continue; + } + agg_update_moments(slot, scratch[0], scratch[1], scratch[2], + static_cast(data[row])); + } + break; + case ir::AggFunc::First: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row) || slot.present()) { + continue; + } + if constexpr (std::is_same_v) { + slot.double_value = data[row]; + } else { + slot.int_value = data[row]; + } + slot.mark_present(); + } + break; + case ir::AggFunc::Last: + for (std::size_t row = start; row < end; ++row) { + if (!valid(row)) { + continue; + } + if constexpr (std::is_same_v) { + slot.double_value = data[row]; + } else { + slot.int_value = data[row]; + } + slot.mark_present(); + } + break; + default: + break; + } + } + + // Flush the open group's key + aggregate values into the output buffers. + void close_group() { + for (std::size_t i = 0; i < group_by_->size(); ++i) { + append_scalar(*out_columns_[i].column, open_key_[i]); + } + for (std::size_t i = 0; i < n_aggs_; ++i) { + ColumnValue& column = *out_columns_[group_by_->size() + i].column; + const AggSlotCore& slot = cur_slots_[i]; + if (track_validity_[i] != 0U) { + out_validity_[i].push_back(chunked_agg_valid(plan_[i].func, slot)); + } + switch (plan_[i].func) { + case ir::AggFunc::Count: + append_scalar(column, ScalarValue{slot.count}); + break; + case ir::AggFunc::Mean: + append_scalar( + column, ScalarValue{slot.count == 0 ? 0.0 + : slot.double_value / + static_cast(slot.count)}); + break; + case ir::AggFunc::Stddev: + append_scalar(column, ScalarValue{agg_finalize_stddev( + slot, cur_scratch_[i * kMomentScratch])}); + break; + case ir::AggFunc::Skew: + append_scalar(column, ScalarValue{agg_finalize_skew( + slot, cur_scratch_[i * kMomentScratch], + cur_scratch_[(i * kMomentScratch) + 1])}); + break; + case ir::AggFunc::Kurtosis: + append_scalar(column, ScalarValue{agg_finalize_kurtosis( + slot, cur_scratch_[i * kMomentScratch], + cur_scratch_[(i * kMomentScratch) + 2])}); + break; + default: // Sum / Min / Max + if (plan_[i].kind == ExprType::Double) { + append_scalar(column, ScalarValue{slot.double_value}); + } else { + append_scalar(column, ScalarValue{slot.int_value}); + } + break; + } + } + ++pending_rows_; + } + + auto take_pending() -> Chunk { + for (std::size_t i = 0; i < n_aggs_; ++i) { + if (track_validity_[i] == 0U || out_validity_[i].empty()) { + continue; + } + bool has_null = false; + for (std::size_t r = 0; r < out_validity_[i].size(); ++r) { + if (!out_validity_[i][r]) { + has_null = true; + break; + } + } + if (has_null) { + out_columns_[group_by_->size() + i].validity = std::move(out_validity_[i]); + } + } + Chunk out; + out.columns = std::move(out_columns_); + if (!out_ordering_.empty()) { + out.set_properties(TableProperties::sorted_by(out_ordering_)); + } + reset_output(); + return out; + } + + OperatorPtr child_; + const std::vector* group_by_; + const std::vector* aggregations_; + const ExecutionContext* exec_; + /// Forwarded verbatim to the hash aggregate fallback -- + /// the sorted stream itself has no fan-out point (it emits group-at-a-time). + physical::AggregateParallelism par_{}; + std::optional columns_; + bool columns_bound_ = false; + + bool decided_ = false; + bool done_ = false; + bool input_eof_ = false; + bool open_ = false; + OperatorPtr fallback_; + + static constexpr std::size_t kEmitThreshold = 8192; + + std::size_t n_aggs_ = 0; + std::vector plan_; + std::vector key_templates_; + std::vector track_validity_; + std::vector out_ordering_; + + // Open-group state. + std::vector cur_slots_; + /// Scratch for the group currently being streamed — 2 doubles per + /// aggregate that declared any (see SlotPlan::scratch_doubles). This + /// operator holds one group at a time, so it needs one group's worth. + /// [m2, m3, m4] per aggregate. Every moment aggregate needs m2 -- the + /// higher ones read it on each update -- so the stride is uniform rather + /// than per-function; this operator keeps one group's worth, not millions. + static constexpr std::size_t kMomentScratch = 3; + std::vector cur_scratch_; + std::vector open_key_; + + // Output buffers for closed groups awaiting emission. + std::vector out_columns_; + std::vector out_validity_; + std::size_t pending_rows_ = 0; +}; + +} // namespace + +auto make_chunked_aggregate_operator(OperatorPtr child, const std::vector* group_by, + const std::vector* aggregations, + const ExecutionContext& exec, + physical::AggregateParallelism parallelism, + std::optional columns) + -> OperatorPtr { + return std::make_unique( + std::move(child), group_by, aggregations, exec, parallelism, std::move(columns)); +} + +} // namespace ibex::runtime diff --git a/src/runtime/aggregate_chunked_internal.hpp b/src/runtime/aggregate_chunked_internal.hpp new file mode 100644 index 00000000..d3b526dd --- /dev/null +++ b/src/runtime/aggregate_chunked_internal.hpp @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +#pragma once + +#include +#include + +#include +#include +#include + +#include "physical_plan.hpp" + +namespace ibex::ir { +struct AggSpec; +struct ColumnRef; +} // namespace ibex::ir + +namespace ibex::runtime { + +/// Construct the adaptive sorted/hash aggregate implementation. Kept as one +/// factory boundary so the complete aggregate family can live in its own +/// translation unit without exposing its state types to the pipeline builder. +[[nodiscard]] auto make_chunked_aggregate_operator( + OperatorPtr child, const std::vector* group_by, + const std::vector* aggregations, const ExecutionContext& exec, + physical::AggregateParallelism parallelism, + std::optional columns) -> OperatorPtr; + +} // namespace ibex::runtime diff --git a/src/runtime/chunked.cpp b/src/runtime/chunked.cpp index a46c85fd..b3353fc5 100644 --- a/src/runtime/chunked.cpp +++ b/src/runtime/chunked.cpp @@ -1,9 +1,11 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright (C) 2026 Bob Jansen -// chunked.cpp — streaming (chunked) operator pipeline: per-chunk operators, -// rank evaluation, extern-call execution, and build_operator plan construction. -// Split out of interpreter.cpp; shared declarations live in interpreter_internal.hpp. +// chunked.cpp — residual streaming operators, rank evaluation, extern-call +// execution, and concrete factories used by the physical and pipeline +// executors. Planning, migrated-plan dispatch, and generic pipeline execution +// live in their respective translation units. Split out of interpreter.cpp; +// shared declarations live in interpreter_internal.hpp. #include #include @@ -63,14 +65,19 @@ #include #endif +#include "aggregate_chunked_internal.hpp" #include "chunk_conversion_internal.hpp" #include "execution_profile_internal.hpp" #include "interpreter_internal.hpp" +#include "join_chunked_internal.hpp" #include "join_internal.hpp" #include "kernel_filter.hpp" #include "kernel_types.hpp" #include "kernel_update.hpp" #include "model_internal.hpp" +#include "packed_key_encoder_internal.hpp" +#include "physical_executor_internal.hpp" +#include "pipeline_executor_internal.hpp" #include "reshape_internal.hpp" #include "runtime_internal.hpp" @@ -130,48 +137,6 @@ struct ChunkIdentity { return chunk; } -// Whether a streamed aggregate slot has enough observations to be non-null. -// Mirrors the materializing aggregate's `agg_result_is_valid`. -auto chunked_agg_valid(ir::AggFunc func, const AggSlotCore& slot) -> bool { - switch (func) { - case ir::AggFunc::Mean: - return slot.count > 0; - case ir::AggFunc::Sum: - case ir::AggFunc::Min: - case ir::AggFunc::Max: - case ir::AggFunc::First: - case ir::AggFunc::Last: - return slot.present(); - case ir::AggFunc::Stddev: - return slot.count >= 2; - case ir::AggFunc::Skew: - return slot.count >= 3; - case ir::AggFunc::Kurtosis: - return slot.count >= 4; - default: // Count - return true; - } -} - -// Whether a streamed aggregate carries a validity bitmap at all (Count never -// produces nulls; the value-bearing aggs may). -auto chunked_agg_tracks_validity(ir::AggFunc func) -> bool { - switch (func) { - case ir::AggFunc::Sum: - case ir::AggFunc::Mean: - case ir::AggFunc::Min: - case ir::AggFunc::Max: - case ir::AggFunc::First: - case ir::AggFunc::Last: - case ir::AggFunc::Stddev: - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - return true; - default: - return false; - } -} - } // namespace /// Per-chunk filter: pulls a chunk from the child, wraps it as a `Table`, @@ -653,7 +618,6 @@ class ChunkedRenameOperator final : public Operator { }; using ir::collect_expr_column_refs; -using ir::is_row_local_update_expr; /// Per-chunk update for row-local field expressions. `build_operator()` only /// routes here when all of the UpdateNode's field expressions are row-local @@ -2179,7 +2143,7 @@ class ChunkedOrderedLimitOperator final : public Operator { push_key_value(key, *entry, row); } group_keys_.push_back(std::move(key)); - group_states_.push_back(GroupState{}); + group_states_.emplace_back(); return static_cast(group_states_.size() - 1); }); } @@ -2379,7 +2343,7 @@ class ChunkedOrderedLimitOperator final : public Operator { /// Encodes a fixed-width multi-column key into one flat integer. /// -/// Shared by `ChunkedDistinctOperator` and `ChunkedAggregateOperator`: both +/// Shared by `ChunkedDistinctOperator` and the hash aggregate state: both /// want the same thing from a multi-column key — a POD that hashes and compares /// in one shot, with no per-row allocation and no per-row string hashing — and /// both then hand it to a partitioned parallel discovery pass. @@ -2390,285 +2354,6 @@ class ChunkedOrderedLimitOperator final : public Operator { /// rather than a template parameter. That is what keeps this from multiplying /// out: the three existing hand-written shapes (one int, two ints, one /// categorical) are all points inside it. -struct PackedKeyEncoder { - // MSVC has no __uint128_t. This is only a packed identity key, so an array - // of words is both portable and avoids pulling a compiler-specific integer - // type into the packed key path. - template - struct PackedWords { - std::array w{}; - - [[nodiscard]] friend auto operator==(const PackedWords&, const PackedWords&) - -> bool = default; - }; - template - struct PackedWordsHash { - auto operator()(const PackedWords& value) const noexcept -> std::size_t { - std::uint64_t acc = 0; - for (const auto word : value.w) { - acc ^= word + 0x9e3779b97f4a7c15ULL + (acc << 6U) + (acc >> 2U); - } - return static_cast(acc); - } - }; - using Packed128 = PackedWords<2>; - using Packed256 = PackedWords<4>; - - /// OR `cell` into the packed key at bit offset `shift`. A cell never spans - /// more than two words because no cell is wider than 64 bits. - template - static void splice(PackedWords& key, std::uint64_t cell, unsigned shift) { - const unsigned word = shift / 64U; - const unsigned off = shift % 64U; - key.w[word] |= cell << off; - // `cell >> 64` is UB, so the carry into the next word is only taken when - // the cell actually straddles the boundary. - if (off != 0 && word + 1 < Words) { - key.w[word + 1] |= cell >> (64U - off); - } - } - - /// The exact inverse of `splice`/the single-word shift in `pack_row`: - /// recover the `width_bits`-wide cell that was spliced in at bit offset - /// `shift`. Only ever needed to decode an ALREADY-PACKED key back into its - /// per-column values (fast-path migration); packing itself never reads a - /// cell back out, so this has no hot-path cost. - template - [[nodiscard]] static auto extract_cell(const Packed& key, unsigned shift, unsigned width_bits) - -> std::uint64_t { - const std::uint64_t mask = - width_bits >= 64U ? ~std::uint64_t{0} : ((std::uint64_t{1} << width_bits) - 1); - if constexpr (std::is_same_v) { - return (key >> shift) & mask; - } else { - const unsigned word = shift / 64U; - const unsigned off = shift % 64U; - std::uint64_t value = key.w[word] >> off; - if (off != 0 && word + 1 < key.w.size()) { - value |= key.w[word + 1] << (64U - off); - } - return value & mask; - } - } - - /// One fixed-width integral key column, resolved to its raw storage and the - /// bit offset it occupies in the packed key. - struct PackCol { - enum class Kind : std::uint8_t { Int64, Date, Ts, Bool, Cat } kind{Kind::Int64}; - const std::int64_t* i64 = nullptr; - const Date* date = nullptr; - const Timestamp* ts = nullptr; - const Column* boolean = nullptr; - const Column* cat = nullptr; - const std::uint32_t* remap = nullptr; ///< local code -> operator-global id - unsigned shift = 0; ///< bit offset of this column's cell in the packed key - }; - struct PackedPlan { - std::vector cols; - unsigned width = 0; ///< total packed width in bytes - }; - - /// Bit width of one packed cell, matching the byte counts - /// `build_packed_layout` accumulates per `PackCol::Kind`. - [[nodiscard]] static auto width_bits_of(PackCol::Kind kind) -> unsigned { - switch (kind) { - case PackCol::Kind::Int64: - case PackCol::Kind::Ts: - return 64U; - case PackCol::Kind::Date: - case PackCol::Kind::Cat: - return 32U; - case PackCol::Kind::Bool: - return 8U; - } - return 0U; - } - - /// Per-key-column interning state for Categorical columns. - /// - /// A categorical code is only meaningful against ITS OWN chunk's dictionary, - /// so packing the raw code would merge two different values that happen to - /// share a code in different chunks. Resolving each dictionary entry to an - /// operator-global id fixes that, and costs one lookup per DICTIONARY ENTRY - /// per chunk rather than one per row: the row loop then reads `remap[code]`, - /// a single array index with no hashing and no allocation at all. - struct CatIntern { - /// Views point into `arena`, whose deque never invalidates references. - robin_hood::unordered_flat_map ids; - std::deque arena; - std::vector remap; ///< rebuilt per chunk, indexed by local code - }; - - /// A key is packable iff every column reduces to a fixed-width INTEGRAL cell - /// whose byte equality equals value equality, with no nulls, and the columns - /// together fit in 32 bytes. - /// - /// Doubles are excluded (-0.0/NaN break byte equality). Strings are excluded - /// because interning one per row would cost the hash lookup this path exists - /// to avoid. Categoricals ARE included: their dictionary is interned once per - /// chunk into operator-global ids (see `CatIntern`), which is what makes a - /// code comparable across chunks. - auto build_packed_key(const std::vector& entries) - -> std::optional { - for (const auto* entry : entries) { - if (entry->validity.has_value()) { - return std::nullopt; - } - } - return build_packed_layout(entries); - } - - /// Same as `build_packed_key`, minus the "no column may carry nulls" check. - /// - /// Used to recover a stable fast path's (kind, shift) layout when a LATER - /// chunk's nulls are exactly what disqualifies `build_packed_key` -- the - /// layout itself does not depend on nullability, only on each column's - /// type and position, which stay fixed for the life of the query once the - /// packed path has been selected. The migration path calls this to learn - /// how to decode the packed keys a prior, null-free chunk already built. - auto build_packed_layout(const std::vector& entries) - -> std::optional { - // Size the interning state ONCE, before any of it is pointed at. - // - // `intern_categorical` hands back `remap.data()`, and `PackCol` holds - // that pointer for the rest of the chunk. Growing `cat_interns_` while - // those pointers are live reallocates the vector, and `CatIntern` holds - // a robin_hood map whose move constructor is not noexcept — so - // `move_if_noexcept` COPIES, `remap` gets a fresh buffer, and column - // 0's pointer is left dangling the moment column 1 is interned. - // - // The symptom was a second categorical key column silently reading - // freed memory: PDS-H q7's `by { supp_nation, cust_nation, l_year }` - // emitted 8 groups instead of 4, the first chunk's four separated from - // the rest, because only the first chunk paid a reallocation. It needed - // multi-chunk input to show at all (`IBEX_CHUNK_ROWS`). - if (cat_interns_.size() < entries.size()) { - cat_interns_.resize(entries.size()); - } - PackedPlan plan; - plan.cols.reserve(entries.size()); - unsigned bytes = 0; - for (std::size_t k = 0; k < entries.size(); ++k) { - const auto& entry = *entries[k]; - PackCol col; - col.shift = bytes * 8; - const ColumnValue& column = *entry.column; - if (const auto* c_int = std::get_if>(&column)) { - col.kind = PackCol::Kind::Int64; - col.i64 = c_int->data(); - bytes += 8; - } else if (const auto* c_date = std::get_if>(&column)) { - col.kind = PackCol::Kind::Date; - col.date = c_date->data(); - bytes += 4; - } else if (const auto* c_ts = std::get_if>(&column)) { - col.kind = PackCol::Kind::Ts; - col.ts = c_ts->data(); - bytes += 8; - } else if (const auto* c_bool = std::get_if>(&column)) { - col.kind = PackCol::Kind::Bool; - col.boolean = c_bool; - bytes += 1; - } else if (const auto* c_cat = std::get_if>(&column)) { - col.remap = intern_categorical(k, *c_cat); - // An empty dictionary would leave the row loop indexing a remap - // that has no entry for any code. Declining here keeps the hot - // loop free of a per-row range check. - if (col.remap == nullptr) { - return std::nullopt; - } - col.kind = PackCol::Kind::Cat; - col.cat = c_cat; - bytes += 4; - } else { - return std::nullopt; - } - if (bytes > sizeof(Packed256)) { - return std::nullopt; - } - plan.cols.push_back(col); - } - plan.width = bytes; - return plan; - } - - /// Resolve chunk-local codes of key column `k` to operator-global ids, - /// returning the remap, or nullptr when the dictionary is empty. Runs once - /// per chunk per categorical key column. - auto intern_categorical(std::size_t k, const Column& cat) -> const std::uint32_t* { - // Never grows `cat_interns_` — `build_packed_key` sized it before any - // caller took a pointer into it, and growing here would dangle those. - auto& state = cat_interns_[k]; - const auto& dict = cat.dictionary(); - const std::size_t dict_size = dict.size(); - if (dict_size == 0) { - return nullptr; - } - state.remap.resize(dict_size); - for (std::size_t code = 0; code < dict_size; ++code) { - const std::string_view value = dict[code]; - if (const auto it = state.ids.find(value); it != state.ids.end()) { - state.remap[code] = it->second; - continue; - } - const auto id = static_cast(state.ids.size()); - // The view must outlive the chunk's dictionary, so the map keys are - // views into this deque rather than into the column. - state.arena.emplace_back(value); - state.ids.emplace(std::string_view{state.arena.back()}, id); - state.remap[code] = id; - } - return state.remap.data(); - } - - /// Pack one row's key. Cheap enough — a handful of array reads and shifts, - /// no hashing and no branch on width — that callers which need the key - /// twice recompute it rather than materialize a buffer. - template - [[nodiscard]] static auto pack_row(const std::vector& cols, std::size_t row) - -> Packed { - Packed key{}; - for (const auto& col : cols) { - std::uint64_t cell = 0; - switch (col.kind) { - case PackCol::Kind::Int64: - cell = static_cast(col.i64[row]); - break; - case PackCol::Kind::Date: - cell = static_cast(col.date[row].days); - break; - case PackCol::Kind::Ts: - cell = static_cast(col.ts[row].nanos); - break; - case PackCol::Kind::Bool: - cell = (*col.boolean)[row] ? 1U : 0U; - break; - case PackCol::Kind::Cat: - cell = col.remap[static_cast(col.cat->code_at(row))]; - break; - } - if constexpr (std::is_same_v) { - key |= cell << col.shift; - } else { - splice(key, cell, col.shift); - } - } - return key; - } - - /// Materialize this chunk's packed key for every row in `[begin, end)`. - template - static void build_keys(const std::vector& cols, std::size_t begin, std::size_t end, - Packed* out) { - for (std::size_t row = begin; row < end; ++row) { - out[row] = pack_row(cols, row); - } - } - - /// Interning state, indexed by key column position (see `CatIntern`). - std::vector cat_interns_; -}; - class ChunkedDistinctOperator final : public Operator { public: ChunkedDistinctOperator(OperatorPtr child, physical::BreakerParallelism dedup_plan) @@ -4129,10445 +3814,617 @@ class ChunkedSemiAntiJoinOperator final : public Operator { std::vector left_cat_matches_; }; -/// The base Scan under `node`, peeled through a chain of Project/Rename/Update -/// wrappers — null when `node` is not (a simple wrapper around) one scan. -/// Deliberately not past Filter: `deferred_probe_scan_of` reuses this peel, -/// and the driver only ever registers a probe scan for exactly the -/// Project/Rename/Update shape it proved eligible, so widening the peel here -/// would silently widen what counts as a deferred probe too. -auto base_scan_of(const ir::Node& node) -> const ir::ScanNode* { - const ir::Node* cur = &node; - while (cur->kind() == ir::NodeKind::Project || cur->kind() == ir::NodeKind::Rename || - cur->kind() == ir::NodeKind::Update) { - if (cur->children().size() != 1 || cur->children().front() == nullptr) { - return nullptr; - } - cur = cur->children().front().get(); - } - if (cur->kind() != ir::NodeKind::Scan) { - return nullptr; - } - return &ir::node_cast(*cur); -} +} // namespace -/// If `right` is a chain of Project/Rename nodes over a Scan whose name the -/// driver registered as a deferred probe scan, return its registration. The -/// driver only registers scans it proved eligible (ir::deferrable_probe_scans: -/// occurs once, feeds exactly this shape), so a hit here IS the eligible -/// position. -struct DeferredProbeScan { - const DeferredScan* scan = nullptr; - const std::string* name = nullptr; ///< scan (instance) name in the plan -}; +auto materialize_operator(OperatorPtr op) -> std::expected { + MaterializeOperator sink{std::move(op)}; + return sink.run(); +} -auto deferred_probe_scan_of(const ir::Node& right, const ExecutionContext& exec) - -> DeferredProbeScan { - if (exec.deferred_scans == nullptr) { - return {}; - } - const auto* scan_node = base_scan_of(right); - if (scan_node == nullptr) { - return {}; - } - const auto& name = scan_node->source_name(); - const auto* scan = exec.deferred_scan(name); - // A probe scan is one with a filter slot to publish build-side bounds - // into. The registry also holds streaming registrations (Phase 1), which - // have no slot and are not this join's to decode. - if (scan == nullptr || scan->filter == nullptr) { - return {}; - } - // Recover the stored key iterator to expose the registry's own name string. - const auto it = exec.deferred_scans->find(name); - return DeferredProbeScan{.scan = scan, .name = &it->first}; +auto distinct_table(const Table& input, const ExecutionContext& exec) + -> std::expected { + // I4 convergence: one implementation, reached through both shapes. The + // whole-table signature survives; the whole-table dedup loop does not. + // + // The source copy is shallow — a `Table` holds shared column handles — so + // this costs a vector of names and pointers, not the data. `distinct` on an + // empty column list still works: the operator passes such a chunk straight + // through, which is what the old `columns.empty()` special case did. + auto source = make_table_source(input); + // The whole table is one chunk, so its row count is exact -- the dedup + // phase's fan-out is fully decided here rather than on the first chunk. + physical::BreakerParallelism dedup_plan = physical::distinct_dedup_parallelism( + {.rows = input.rows(), .source = physical::RowEstimate::Source::ChildExact}); + const std::size_t pool_size = exec.can_fan_out() ? process_worker_pool().size() : 0; + physical::resolve_breaker_parallelism(dedup_plan, exec, pool_size); + return materialize_operator( + std::make_unique(std::move(source), dedup_plan)); } -/// The end of a hash chain. At namespace scope because the index and the -/// operator that probes it are no longer the same type, so the sentinel belongs -/// to neither alone; `ChunkedInnerJoinOperator::kNil` aliases it. -inline constexpr std::size_t kJoinNil = std::numeric_limits::max(); +auto is_streamable_inner_join(const ir::JoinNode& join) -> bool { + // `nulls equal` goes to the materialized join, which implements the policy. + // The streaming operators hash and probe on their own and would each need + // the same null tagging; sending the opt-in case to the one implementation + // that has it keeps a single definition of the semantics. + return join.kind() == ir::JoinKind::Inner && !join.predicate().has_value() && + join.keys().size() == 1 && join.null_match() == ir::NullMatch::Never && + !join.expect().asserts_anything() && join.take() == ir::MatchSelection::All; +} -/// A hash-index head table split into partitions by key hash. -/// -/// Every key belongs to exactly one partition, so P workers can fill P -/// partitions with no shared writes, no locks, and -- unlike per-worker maps -- -/// no merge afterwards. That is what makes a hash build morsel-parallel, and -/// it is the reason this type exists: `build_join_hash_index` is one serial -/// loop, and on TPC-H q21 it spends 40 ms hashing 1.29M rows inside a 75 ms -/// query (measured 2026-08-25, see plans/kernel-pipeline-execution-plan.md, -/// "Where join time actually goes"). +/// Two-fixed-width-int-key inner join (plans/parallelism-overview.md's +/// "stream multi-key joins" item; TPC-H q09's `lineitem` join is the +/// motivating case). Same structural gate as `is_streamable_inner_join` +/// plus a static schema check -- both keys, on both sides, must be provably +/// `Int64` -- so an ineligible pair (a string key, an unascribed/Unknown +/// schema) falls through to the materialized-call fallback (`interpret_node`'s +/// `Join` branch) exactly as it does today, never into a code path that could +/// fail at runtime. +/// `ChunkedInnerJoinOperator::initialize_pair` re-checks the actual runtime +/// column type regardless -- this is a routing optimization, not the sole +/// guarantee of correctness. +/// Whether every aggregation in `agg` can be computed incrementally, and so +/// streamed rather than materialized. /// -/// `partition_count == 1` is exactly the single-map behaviour this replaced, -/// bit for bit: one partition, mask 0, every key landing in `parts[0]`. -/// Partitioning the TYPE and filling it in parallel are deliberately separate -/// steps -- the first cannot change a result, so anything the second breaks is -/// unambiguously the second's fault. -template , class Eq = std::equal_to> -struct PartitionedHeads { - using Map = robin_hood::unordered_flat_map; - /// Always a power of two, so `part_of` is a mask rather than a modulo. - std::vector parts{1}; - std::size_t mask = 0; - - /// Size to `count` partitions (rounded down to a power of two, at least 1). - void partition(std::size_t count) { - std::size_t p = 1; - while (p * 2 <= count) { - p *= 2; +/// Named and shared for the same reason the join gates were: the physical +/// planner has to relay this rather than restate it. Reimplementing a +/// multi-clause eligibility test in the planner is what made it wrong about +/// two-key joins within an hour of being written. +auto aggregate_is_streamable(const ir::AggregateNode& agg) -> bool { + return std::ranges::all_of(agg.aggregations(), [](const ir::AggSpec& spec) { + switch (spec.func) { + case ir::AggFunc::Count: + case ir::AggFunc::Sum: + case ir::AggFunc::Min: + case ir::AggFunc::Max: + case ir::AggFunc::Mean: + case ir::AggFunc::Stddev: + case ir::AggFunc::Skew: + case ir::AggFunc::Kurtosis: + case ir::AggFunc::First: + case ir::AggFunc::Last: + // First/Last: the operators themselves gate by column type + // (numeric, string, categorical stream; Date/Timestamp fall to + // the hash operator's error path -- unreachable in practice + // since aggregation on those types is rejected upstream of the + // chunked path entirely, same as every other agg func). + return true; + default: + // Median/Quantile need all values; Ewma is row-order coupled -- + // these stay on the materializing path. + return false; } - parts.assign(p, Map{}); - mask = p - 1; - } - - [[nodiscard]] auto partition_count() const noexcept -> std::size_t { return parts.size(); } + }); +} - [[nodiscard]] auto part_of(const Key& key) const noexcept -> std::size_t { - return mask == 0 ? 0 : (Hash{}(key)&mask); - } +auto is_streamable_semi_anti_join(const ir::JoinNode& join) -> bool { + return (join.kind() == ir::JoinKind::Semi || join.kind() == ir::JoinKind::Anti) && + !join.predicate().has_value() && join.keys().size() == 1 && + join.null_match() == ir::NullMatch::Never && !join.expect().asserts_anything() && + join.take() == ir::MatchSelection::All; +} - /// Reserve for `n` build rows. Split across partitions, since a key can - /// only land in one of them. - void reserve(std::size_t n) { - const std::size_t per = (n / parts.size()) + 1; - for (auto& part : parts) { - part.reserve(per); - } +auto is_streamable_pair_int_join(const ir::JoinNode& join) -> bool { + if (join.kind() != ir::JoinKind::Inner || join.predicate().has_value() || + join.keys().size() != 2 || join.null_match() != ir::NullMatch::Never || + join.expect().asserts_anything() || join.take() != ir::MatchSelection::All) { + return false; } - - /// Insert `row` as the head for `key` if absent. Returns a pointer to the - /// stored head (never null) and whether it was newly inserted, so a caller - /// that loses the race to an earlier row can chain onto what is there. - auto try_emplace(const Key& key, std::size_t row) -> std::pair { - auto [it, inserted] = parts[part_of(key)].try_emplace(key, row); - return {&it->second, inserted}; + const ir::SchemaInfo left_schema = ir::infer_schema(*join.children()[0]); + const ir::SchemaInfo right_schema = ir::infer_schema(*join.children()[1]); + if (!left_schema.is_known() || !right_schema.is_known()) { + return false; } - - /// The head row for `key`, or `kJoinNil` when the build side has none. - [[nodiscard]] auto find_head(const Key& key) const -> std::size_t { - const auto& part = parts[part_of(key)]; - const auto it = part.find(key); - return it == part.end() ? kJoinNil : it->second; + // NOLINTNEXTLINE(readability-use-anyofallof) + for (const ir::JoinKey& key : join.keys()) { + const ir::SchemaField* lf = left_schema.find(key.left); + const ir::SchemaField* rf = right_schema.find(key.right); + if (lf == nullptr || rf == nullptr || !lf->type.has_value() || !rf->type.has_value() || + *lf->type != ir::ColumnType::Int64 || *rf->type != ir::ColumnType::Int64) { + return false; + } } -}; + return true; +} -/// Everything a hash build produces and a hash probe consumes: the chained -/// index over one side's key column, plus what the probe needs to interpret it. -/// -/// This is the barrier between Phase 4's `HashBuild` and `HashProbe`, stated as -/// a type. A probe holds it as `shared_ptr`: it cannot -/// write to what a build produced, and one build can feed several probes. -/// -/// What is deliberately NOT here: the categorical code -> head table. It is -/// derived from the PROBE chunk's dictionary and rebuilt per chunk, so it is -/// the probing operator's state (`probe_code_heads_`). Holding it here is what -/// made the build state mutable during probing. -struct JoinHashIndex { - ExprType key_kind = ExprType::Int; - /// False once any key repeats: the probe can skip chain walking when a - /// build side is unique. - bool unique = true; - /// Row -> next row with the same key, `kJoinNil` at the end of a chain. - std::vector chain_next; - /// Head row per key, one map per key representation. - PartitionedHeads i64_heads; - PartitionedHeads f64_heads; - PartitionedHeads bool_heads; - PartitionedHeads date_heads; - PartitionedHeads ts_heads; - PartitionedHeads string_heads; - - /// Two-fixed-width-int-key path: both key values pack into one struct, - /// injective with no knowledge of their domains -- same trick as the - /// aggregate's own `PairIntKey`. - struct PairKey { - std::uint64_t a = 0; - std::uint64_t b = 0; - [[nodiscard]] friend auto operator==(const PairKey&, const PairKey&) -> bool = default; - }; - struct PairKeyHash { - auto operator()(const PairKey& key) const noexcept -> std::size_t { - std::uint64_t h = key.a * 0x9e3779b97f4a7c15ULL; - h ^= key.b + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); - return static_cast(h); - } - }; - PartitionedHeads pair_heads; +auto inner_join_table(const Table& left, const Table& right, const std::vector& keys, + const ir::JoinSuffixPolicy& suffix, + const std::vector& pending_order, const ExecutionContext& exec) + -> std::expected { + // I4 convergence: keep the materialized signature used by interpret_node, + // but run the same hash join that build_operator selects for this exact + // semantic subset. Table sources copy only their column handles. + auto source = make_table_source(left); + return materialize_operator(make_chunked_inner_join_operator( + std::move(source), right, &keys, exec, suffix, &pending_order, + physical_executor_detail::resolved_join_parallelism(exec))); +} - /// Borrowed from the build table; null when the key column has no nulls. A - /// null key matches nothing, so null build rows are never indexed and null - /// probe rows are never looked up. - const ValidityBitmap* validity = nullptr; -}; +namespace { -auto detect_join_key_kind(const ColumnValue& col, ExprType& out) -> std::optional { - if (std::holds_alternative>(col)) { - out = ExprType::Int; - } else if (std::holds_alternative>(col)) { - out = ExprType::Double; - } else if (std::holds_alternative>(col)) { - out = ExprType::Bool; - } else if (std::holds_alternative>(col)) { - out = ExprType::Date; - } else if (std::holds_alternative>(col)) { - out = ExprType::Timestamp; - } else if (std::holds_alternative>(col) || - std::holds_alternative>(col)) { - out = ExprType::String; - } else { - return "ChunkedInnerJoinOperator: unsupported key type"; +// The single choke point for "build this subtree, then immediately drain it +// to a whole Table" — every call site in this file that needs a materialized +// side (a join's build/probe side, an update's input, an aggregate's fused +// join operand, ...) with no downstream consumer to overlap the build with +// routes through here, rather than each hand-rolling its own +// `build_operator` + `materialize_operator` pair. One place to reason about +// this pattern instead of N independently-drifting copies. +auto materialize_row_local(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected { + auto op = build_operator(node, registry, scalars, externs, exec, model_out); + if (!op.has_value()) { + return std::unexpected(std::move(op.error())); } - return std::nullopt; + return materialize_operator(std::move(op.value())); } -/// The hash build. Chains each build row to the next row carrying the same key, -/// iterating in reverse so a chain walks forward during the probe -- which is -/// what makes the streamed output match the nested-loop inner join's ordering. -/// -/// A null key matches nothing, not even another null (SQL / Polars). So a -/// null-keyed build row is never indexed, and a null-keyed probe row is never -/// looked up. Both halves are needed: a null cell holds the type's zero value, -/// so a null probe key would otherwise find a genuine `0`. -/// Fill one `PartitionedHeads` from `n` build rows, serially when it has one -/// partition and over the worker pool when it has more. -/// -/// The two paths must produce a bit-identical index, and the reason they do is -/// worth stating rather than trusting. The serial build walks rows from `n-1` -/// down to 0, so a key's head is its LOWEST row and its chain ascends. The -/// parallel build scatters rows into partitions keeping them ascending within -/// each, then walks each partition's slice in reverse -- and because every row -/// carrying a given key lands in that key's one partition, reverse order -/// within the partition is reverse order within the key. Same head, same -/// chain, in any partition count. Output row order is a join contract -/// (SPEC.md 5.6 leaves it open, but `emit_swapped` and the chained probe both -/// depend on the chain's direction), so this is not a detail. -/// -/// Writes are disjoint by construction: `chain_next[row]` is written only by -/// the worker owning that row's partition, and each partition's map is its -/// own. No locks, and no merge afterwards -- which is the whole reason the -/// head table is partitioned rather than built per-worker and combined. -template -void fill_partitioned_heads(Heads& heads, std::size_t n, const KeyAt& key_at, const IsNull& is_null, - std::vector& chain_next, bool& unique) { - const auto insert = [&](std::size_t row, bool& dup) { - auto [head, inserted] = heads.try_emplace(key_at(row), row); - if (!inserted) { - chain_next[row] = *head; - *head = row; - dup = true; - } - }; - if (heads.partition_count() == 1) { - heads.reserve(n); - bool dup = false; - for (std::size_t r = n; r-- > 0;) { - if (is_null(r)) { - continue; +// The relational inputs of a materialized-call fallback node -- the subtrees +// `build_materialized_fallback` may build ahead through the physical path. For +// most kinds these are the direct children. Two shapes carry a child that is +// *not* an independent relational input and must not be built standalone: +// - `Window`'s child is an `update` clause; only `interpret_node`'s Window +// case may evaluate it (it needs the window duration). The real input is +// the update's own child. +// - `Stream`'s child is a per-buffer transform template over `__stream_input__` +// and has no meaning outside the stream loop. +// A kind not listed here (or one whose children are template/expression nodes) +// gets no pre-build: `interpret_node` evaluates it whole, which is the prior +// behaviour. +auto fallback_relational_inputs(const ir::Node& node) -> std::vector { + std::vector inputs; + switch (node.kind()) { + case ir::NodeKind::Melt: + case ir::NodeKind::Dcast: + case ir::NodeKind::Columns: + case ir::NodeKind::Cov: + case ir::NodeKind::Corr: + case ir::NodeKind::Transpose: + case ir::NodeKind::Matmul: + case ir::NodeKind::Resample: + case ir::NodeKind::Model: + case ir::NodeKind::AsTimeframe: + case ir::NodeKind::Update: + case ir::NodeKind::Join: + inputs.reserve(node.children().size()); + for (const auto& child : node.children()) { + inputs.push_back(child.get()); } - insert(r, dup); - } - unique = unique && !dup; - return; - } - - const std::size_t parts = heads.partition_count(); - auto& pool = process_worker_pool(); - const std::size_t ranges = std::min(pool.size(), parts); - const std::size_t grain = (n + ranges - 1) / ranges; - heads.reserve(n); - - // Pass 1: which partition each row belongs to, and how many rows each - // (range, partition) pair contributes. Null-keyed rows are never indexed, - // so they are given no partition and never counted. - std::vector part_of_row(n, 0); - std::vector indexed(n, 0); - std::vector counts(ranges * parts, 0); - { - auto batch = pool.submit(ranges, [&](std::size_t r) { - const std::size_t begin = r * grain; - const std::size_t end = std::min(n, begin + grain); - std::size_t* row_counts = counts.data() + (r * parts); - for (std::size_t row = begin; row < end; ++row) { - if (is_null(row)) { - continue; - } - const std::size_t part = heads.part_of(key_at(row)); - part_of_row[row] = static_cast(part); - indexed[row] = 1; - ++row_counts[part]; + break; + case ir::NodeKind::Window: + if (!node.children().empty() && !node.children().front()->children().empty()) { + inputs.push_back(node.children().front()->children().front().get()); } - }); - batch.wait(); + break; + default: + break; } + return inputs; +} - std::vector offsets(ranges * parts, 0); - std::vector part_begin(parts + 1, 0); - { - std::size_t running = 0; - for (std::size_t p = 0; p < parts; ++p) { - part_begin[p] = running; - for (std::size_t r = 0; r < ranges; ++r) { - offsets[(r * parts) + p] = running; - running += counts[(r * parts) + p]; - } +// The materialized-call fallback for every node kind `plan_physical` does not +// migrate (reshape, stats, window, non-row-local update, materializing join, +// matmul, model, ...). `interpret_node` owns the per-kind semantics; this only +// makes sure the breaker's *inputs* still go through the physical path: each +// relational input is built and drained via `build_operator` (fused parallel +// scan, projection pushdown, streaming join), then `interpret_node` runs over +// the node with those inputs handed back pre-built through +// `pre_materialized_children` -- so a `Filter`/`Project` feeding the breaker is +// not re-evaluated whole-table and serial. `interpret_node` still recurses for +// anything deeper, and for any kind `fallback_relational_inputs` leaves empty. +auto build_materialized_fallback(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected { + const std::vector inputs = fallback_relational_inputs(node); + + std::vector
built; + built.reserve(inputs.size()); + for (const ir::Node* input : inputs) { + auto table = materialize_row_local(*input, registry, scalars, externs, exec, model_out); + if (!table.has_value()) { + return std::unexpected(std::move(table.error())); } - part_begin[parts] = running; + built.push_back(std::move(table.value())); } - - // Pass 2: scatter. Ranges are laid out in ascending order within each - // partition and each range walks ascending, so a partition's slice is - // ascending in row index -- which is what pass 3 reverses. - std::vector scatter_rows(part_begin[parts]); - { - auto batch = pool.submit(ranges, [&](std::size_t r) { - const std::size_t begin = r * grain; - const std::size_t end = std::min(n, begin + grain); - std::size_t* cursor = offsets.data() + (r * parts); - for (std::size_t row = begin; row < end; ++row) { - if (indexed[row] == 0) { - continue; - } - scatter_rows[cursor[part_of_row[row]]++] = row; - } - }); - batch.wait(); + // `built` is not resized past this point, so the addresses stay valid for + // the `interpret_node` call below. + std::vector> handback; + handback.reserve(inputs.size()); + for (std::size_t i = 0; i < inputs.size(); ++i) { + handback.emplace_back(inputs[i], &built[i]); } + ExecutionContext local = exec; + local.pre_materialized_children = &handback; - // Pass 3: one worker per partition, claimed dynamically. - std::vector dup(parts, 0); - { - std::atomic cursor{0}; - auto batch = pool.submit(std::min(pool.size(), parts), [&](std::size_t) { - for (std::size_t p = cursor.fetch_add(1, std::memory_order_relaxed); p < parts; - p = cursor.fetch_add(1, std::memory_order_relaxed)) { - bool part_dup = false; - for (std::size_t i = part_begin[p + 1]; i-- > part_begin[p];) { - insert(scatter_rows[i], part_dup); - } - dup[p] = part_dup ? 1 : 0; - } - }); - batch.wait(); - } - for (const char d : dup) { - unique = unique && d == 0; + auto table = interpret_node(node, registry, scalars, externs, local, model_out); + if (!table.has_value()) { + return std::unexpected(std::move(table.error())); } + return make_table_source(std::move(table.value())); } -auto build_join_hash_index(const Table& build_side, const std::string& key_name, ExprType key_kind, - std::size_t partitions) -> std::expected { - const ColumnValue* key = build_side.find(key_name); - if (key == nullptr) { - return std::unexpected("join key not found in build side: " + key_name); - } - JoinHashIndex index; - index.key_kind = key_kind; - const auto* build_entry = build_side.find_entry(key_name); - index.validity = build_entry != nullptr && build_entry->validity.has_value() - ? &*build_entry->validity - : nullptr; - const std::size_t n = build_side.rows(); - index.chain_next.assign(n, kJoinNil); - - const auto is_null = [&index](std::size_t row) noexcept { - return index.validity != nullptr && !(*index.validity)[row]; - }; - const auto build_scalar = [&](const ColT& col, Heads& heads) { - const auto* data = col.data(); - heads.partition(partitions); - fill_partitioned_heads( - heads, col.size(), [data](std::size_t r) { return data[r]; }, is_null, index.chain_next, - index.unique); - }; +template - if (key_kind == ExprType::Int) { - const auto* col = std::get_if>(key); - if (col == nullptr) - return std::unexpected("inner join: build-side key type mismatch"); - build_scalar(*col, index.i64_heads); - } else if (key_kind == ExprType::Double) { - const auto* col = std::get_if>(key); - if (col == nullptr) - return std::unexpected("inner join: build-side key type mismatch"); - build_scalar(*col, index.f64_heads); - } else if (key_kind == ExprType::Bool) { - const auto* col = std::get_if>(key); - if (col == nullptr) - return std::unexpected("inner join: build-side key type mismatch"); - // No `data()` on a packed bool column, so this is the one - // representation `build_scalar` cannot serve. Left unpartitioned: a - // bool key has two values, so partitioning can only leave every - // partition but two empty. - fill_partitioned_heads( - index.bool_heads, n, [col](std::size_t r) { return (*col)[r]; }, is_null, - index.chain_next, index.unique); - } else if (key_kind == ExprType::Date) { - const auto* col = std::get_if>(key); - if (col == nullptr) - return std::unexpected("inner join: build-side key type mismatch"); - build_scalar(*col, index.date_heads); - } else if (key_kind == ExprType::Timestamp) { - const auto* col = std::get_if>(key); - if (col == nullptr) - return std::unexpected("inner join: build-side key type mismatch"); - build_scalar(*col, index.ts_heads); - } else if (key_kind == ExprType::String) { - index.string_heads.partition(partitions); - if (const auto* c_cat = std::get_if>(key)) { - const auto& dict = c_cat->dictionary(); - fill_partitioned_heads( - index.string_heads, n, - [c_cat, &dict](std::size_t r) { - return std::string_view{dict[static_cast(c_cat->code_at(r))]}; - }, - is_null, index.chain_next, index.unique); - } else if (const auto* c_str = std::get_if>(key)) { - fill_partitioned_heads( - index.string_heads, n, - [c_str](std::size_t r) { return std::string_view{(*c_str)[r]}; }, is_null, - index.chain_next, index.unique); - } else { - return std::unexpected("inner join: build-side key type mismatch"); - } - } - return index; -} - -/// The two-key hash build: same chain-of-equal-rows convention as -/// `build_join_hash_index`, over a packed pair of Int64 keys. A row with either -/// key null is never indexed -- null never matches, not even another null. -auto build_join_pair_index(const Column& col0, const Column& col1, - const ValidityBitmap* v0, const ValidityBitmap* v1, - std::size_t partitions) -> JoinHashIndex { - JoinHashIndex index; - index.key_kind = ExprType::Int; - const std::size_t n = col0.size(); - index.chain_next.assign(n, kJoinNil); - const auto* d0 = col0.data(); - const auto* d1 = col1.data(); - index.pair_heads.partition(partitions); - fill_partitioned_heads( - index.pair_heads, n, - [d0, d1](std::size_t r) { - return JoinHashIndex::PairKey{.a = static_cast(d0[r]), - .b = static_cast(d1[r])}; - }, - [v0, v1](std::size_t r) { - return (v0 != nullptr && !(*v0)[r]) || (v1 != nullptr && !(*v1)[r]); - }, - index.chain_next, index.unique); - return index; -} - -/// Which side of a hash join carries the index. -/// -/// The choice is made at RUN time, from measured row counts, and this enum is -/// what makes it a VALUE rather than a shape encoded across three operator -/// members (`mode_`, `probe_op_`, `left_table_`). A build phase -/// can decide it without knowing who will probe, which is what a separately -/// scheduled `HashBuild` needs; see plans/kernel-pipeline-execution-plan.md, -/// "The build-side choice does not block the split" -- the physical plan does -/// not have to name the side statically, because the pipeline that scans the -/// other side is constructed after this phase has already run. -enum class JoinOrientation : std::uint8_t { - BuildRight, ///< index the right side, stream left chunks through it - BuildLeft, ///< index the left side, scan the right once in probe order -}; - -/// Everything a hash build phase decides and produces. Which table to stream -/// and which mode to run in are the caller's derivations from these two, not -/// the build's business. -struct JoinBuildOutcome { - std::shared_ptr index; - JoinOrientation orientation = JoinOrientation::BuildRight; -}; - -/// Build the index over one named side. The primitive both orientation -/// decisions below reduce to, and the only place a `JoinHashIndex` becomes -/// shared and const. -auto build_join_side(const Table& side, const std::string& key_name, ExprType key_kind, - JoinOrientation orientation, std::size_t partitions) - -> std::expected { - auto built = build_join_hash_index(side, key_name, key_kind, partitions); - if (!built.has_value()) { - return std::unexpected(std::move(built.error())); - } - return JoinBuildOutcome{.index = std::make_shared(std::move(*built)), - .orientation = orientation}; -} - -/// The single-key build phase over two already-materialized sides: choose an -/// orientation, build that side's index, return both. Reads and writes no -/// operator state, so the same call serves a join operator and a `HashBuild` -/// that has no probe attached yet. -/// -/// `order_preservation_pays` arrives as a decided bool because it answers a -/// question about the join's OUTPUT plan -- would declining to swap deliver a -/// pending `order` for free -- which is the caller's to answer, not the -/// build's. It is only ever consulted when swapping was otherwise preferred. -auto choose_and_build_single_key(const Table& left, const Table& right, const std::string& left_key, - const std::string& right_key, ExprType key_kind, - bool order_preservation_pays, std::size_t partitions) - -> std::expected { - // Swapping indexes the smaller (left) side and scans the right, which - // gives up left-row order. When an `order` above this join wants exactly - // the order the left already carries, declining to swap delivers it and - // that whole sort disappears -- worth a larger index, but only while - // "larger" stays modest, since the index is probed once per row of the - // other side. The same trade is made in join.cpp. - if (left.rows() < right.rows() && !order_preservation_pays) { - return build_join_side(left, left_key, key_kind, JoinOrientation::BuildLeft, partitions); - } - return build_join_side(right, right_key, key_kind, JoinOrientation::BuildRight, partitions); -} - -/// One side's two Int64 key columns and their validity, or the error a join -/// reports for them. `side_name` is "left" or "right" only so the message -/// keeps naming the side the caller was asking about. -struct PairKeyColumns { - const Column* col0 = nullptr; - const Column* col1 = nullptr; - const ValidityBitmap* v0 = nullptr; - const ValidityBitmap* v1 = nullptr; -}; - -auto pair_key_columns(const Table& side, const std::string& name0, const std::string& name1, - std::string_view side_name) -> std::expected { - const ColumnValue* key0 = side.find(name0); - if (key0 == nullptr) { - return std::unexpected("join key not found in " + std::string(side_name) + - " table: " + name0); - } - const ColumnValue* key1 = side.find(name1); - if (key1 == nullptr) { - return std::unexpected("join key not found in " + std::string(side_name) + - " table: " + name1); - } - PairKeyColumns out; - out.col0 = std::get_if>(key0); - out.col1 = std::get_if>(key1); - if (out.col0 == nullptr || out.col1 == nullptr) { - return std::unexpected( - "ChunkedInnerJoinOperator: two-key join currently requires both keys to be Int64"); - } - const auto* entry0 = side.find_entry(name0); - const auto* entry1 = side.find_entry(name1); - out.v0 = entry0 != nullptr && entry0->validity.has_value() ? &*entry0->validity : nullptr; - out.v1 = entry1 != nullptr && entry1->validity.has_value() ? &*entry1->validity : nullptr; - return out; -} - -/// The two-Int64-key build phase. Same contract as the single-key one, with a -/// simpler decision: no pending-order trade exists on this path, so the -/// smaller side is indexed outright. Validates the right side first, which is -/// the order the operator already checked in, so consolidating the two -/// previously separate blocks cannot change which error a caller sees. -auto choose_and_build_pair(const Table& left, const Table& right, const ir::JoinKey& k0, - const ir::JoinKey& k1, std::size_t partitions) - -> std::expected { - auto right_keys = pair_key_columns(right, k0.right, k1.right, "right"); - if (!right_keys.has_value()) { - return std::unexpected(std::move(right_keys.error())); - } - if (left.rows() <= right.rows()) { - auto left_keys = pair_key_columns(left, k0.left, k1.left, "left"); - if (!left_keys.has_value()) { - return std::unexpected(std::move(left_keys.error())); - } - return JoinBuildOutcome{ - .index = std::make_shared(build_join_pair_index( - *left_keys->col0, *left_keys->col1, left_keys->v0, left_keys->v1, partitions)), - .orientation = JoinOrientation::BuildLeft}; - } - return JoinBuildOutcome{ - .index = std::make_shared(build_join_pair_index( - *right_keys->col0, *right_keys->col1, right_keys->v0, right_keys->v1, partitions)), - .orientation = JoinOrientation::BuildRight}; -} - -/// The probe half of a hash join: everything that consumes a `JoinHashIndex` -/// and turns probe-side rows into join output rows. It owns no build, which is -/// the point -- Phase 4's `HashProbe` has to be able to exist next to a build -/// it did not run, and an operator class that also decides which side to index -/// cannot be that. -/// -/// Held by the operator today rather than scheduled on its own. What the -/// extraction buys now is that the probe's state is enumerable: an index, the -/// join's two output-name plans, the per-worker scratch, and the probe chunk's -/// own validity/dictionary. Nothing else in the join can reach it, and it can -/// reach nothing else in the join. -struct JoinProbe { - const std::vector* keys_ = nullptr; - const ExecutionContext* exec_ = nullptr; - /// The `probe` phase's parallelism, resolved by the caller and set through - /// `ChunkedInnerJoinOperator::bind_probe`. `probe_parallel_workers` reads it - /// rather than re-deriving the floor and the cap. Travels with every copy of - /// this struct (per-worker probes, `JoinProbeOperator`). - physical::BreakerParallelism probe_plan_{}; - ir::JoinSuffixPolicy suffix_; - /// The join's right table, owned jointly and immutably. - /// - /// Shared rather than borrowed because a probe has to be able to outlive - /// the operator that ran the build, and because several probes -- one per - /// morsel worker -- have to be able to read one build side at once. A raw - /// pointer into a member could do neither, and depended on the operator - /// never reallocating the table it points at. Null until the build phase - /// resolves it, which on the deferred path is only after the scan runs. - std::shared_ptr right_; - /// What the build phase produced. Immutable, and shareable: this is the - /// only thing the probe needs from a build. - std::shared_ptr index_; - /// True when the join keys are the two-Int64 pair shape. - bool pair_mode_ = false; - - /// Reset per probe chunk. - const ValidityBitmap* probe_validity_ = nullptr; - /// Probe-side, not build-side: the probe chunk's dictionary code -> build - /// chain head (`kJoinNil` = no match), rebuilt per chunk by - /// `resolve_categorical_heads`. - std::vector probe_code_heads_; - - /// One worker's slice of a parallel probe. Members so the vectors keep - /// their capacity across chunks instead of reallocating per probe. - struct ProbePart { - std::vector li; - std::vector ri; - }; - /// One matching probe row in swapped mode: the right row and the head of - /// the left chain it hit. Phase 2 replays these instead of re-probing. - struct SwappedHit { - std::size_t rrow; - std::size_t head; ///< first left row in the chain for this key - }; - /// One worker's slice of a swapped-mode phase 1. A member for the same - /// reason as `ProbePart`: capacity survives across chunks. - struct SwappedPart { - std::vector hits; - std::size_t total = 0; ///< output rows this part's chains expand to - }; - std::vector probe_parts_; - std::vector swapped_parts_; - std::vector part_offsets_; - std::vector right_emit_idx_; - std::vector right_emit_names_; - std::vector left_emit_names_; - bool right_emit_ready_ = false; - - static constexpr std::size_t kNil = kJoinNil; - - /// The build this probe reads. Never null once a build side has been - /// chosen; the `Precomputed` mode returns before one exists. - [[nodiscard]] auto index() const noexcept -> const JoinHashIndex& { return *index_; } - - [[nodiscard]] auto probe_is_null(std::size_t row) const noexcept -> bool { - return probe_validity_ != nullptr && !(*probe_validity_)[row]; - } - - // Which right columns this join emits, and under which names. Both come - // from the shared planner (ir/join_output.hpp), so the chunked route lands - // on the same output schema as the materialized route and IR inference. - // The left column names are identical for every chunk, so the plan is - // computed once from the first assembled chunk. - - auto setup_right_emit_schema(const Table& left_side) -> std::expected { - auto planned = - ir::plan_join_output(ir::JoinKind::Inner, *keys_, table_column_names(left_side), - table_column_names(*right_), suffix_); - if (!planned.has_value()) { - return std::unexpected(std::move(planned.error())); - } - const std::vector& plan = *planned; - // A suffix clause renames the *left* side of a collision too, so the - // left names come from the plan as well; taking them from the chunk - // would keep the pre-rename spelling. - left_emit_names_.reserve(left_side.columns.size()); - for (const auto& column : plan) { - if (column.side == ir::JoinOutputSide::Left) { - left_emit_names_.push_back(column.name); - } - } - right_emit_idx_.reserve(plan.size() - left_side.columns.size()); - right_emit_names_.reserve(plan.size() - left_side.columns.size()); - for (const auto& column : plan) { - if (column.side != ir::JoinOutputSide::Right) { - continue; - } - right_emit_idx_.push_back(column.source_index); - right_emit_names_.push_back(column.name); - } - right_emit_ready_ = true; - return {}; - } - - /// Run `body(begin, end, li, ri)` over the probe rows across workers, - /// concatenating each range's output in range order. Returns false when the - /// parallel path declines and the caller should run its serial loop. - /// - /// **This is the join's only parallel axis, and it is the whole of it.** - /// The build side is a shared read-only hash index — `heads`, `index().chain_next` - /// — so probing it concurrently needs no locking at all, and the build - /// itself is not worth threading: it is 1.5% of q10 against the probe and - /// output assembly's ~15%. - /// - /// **Per-worker output rather than count-then-fill.** The obvious shape is - /// to count matches per range, prefix-sum, then have each worker write its - /// slice — but that probes the hash table TWICE per row, and a redundant - /// cache-missing lookup per probe row is exactly the cost `emit_swapped` - /// was restructured to avoid (q03 probes 3.2M lineitems to emit ~30K rows). - /// Each worker appends to its own vectors instead and they are concatenated - /// afterwards: one memcpy of two size_t arrays, against one hash probe per - /// row saved. - /// - /// Order is exactly the serial order — ranges are contiguous and visited in - /// order, and each range appends in row order — so the output is - /// byte-identical however the workers interleave. - /// The shared admission gate for every parallel probe axis: how many - /// workers a probe over `n` rows may fan out to, or 0 to decline and run - /// the caller's serial loop. - /// - /// The floor (`1U << 14U`) and the worker cap (`min(compute_budget, pool, - /// 64)`) are the `probe` phase of the physical plan - /// (`physical::join_probe_parallelism`), read from `probe_plan_`. - /// src/runtime/PARALLELISM.md, "Target: parallelism as a plan decision". - /// The checks that stay here are the ones only the operator can make: - /// `parallel_join_probe` (a feature toggle, kept operator-side like the hash - /// build's `IBEX_JOIN_BUILD_SERIAL`), nesting (`on_worker_pool_thread` -- no - /// nested pool submissions), and whether *this chunk* cleared the floor (a - /// streamed probe side's per-chunk row count is not a plan-time fact). - [[nodiscard]] auto probe_parallel_workers(std::size_t n) const -> std::size_t { - if (exec_ == nullptr || !exec_->parallel_join_probe || on_worker_pool_thread() || - probe_plan_.decline != physical::FanOutDecline::None || probe_plan_.worker_cap < 2 || - n < probe_plan_.row_floor) { - return 0; - } - return probe_plan_.worker_cap; - } - - template - auto probe_ranges_parallel(std::size_t n, std::vector& li, - std::vector& ri, const Body& body) -> bool { - const std::size_t workers = probe_parallel_workers(n); - if (workers == 0) { - return false; - } - auto& pool = process_worker_pool(); - const std::size_t grain = (n + workers - 1) / workers; - probe_parts_.resize(workers); - { - auto batch = pool.submit(workers, [&](std::size_t w) { - auto& part = probe_parts_[w]; - part.li.clear(); - part.ri.clear(); - const std::size_t begin = w * grain; - const std::size_t end = std::min(n, begin + grain); - if (begin >= end) { - return; - } - // Reserve for the common case of roughly one match per row; - // a fan-out join grows past it, which is what a vector is for. - part.li.reserve(end - begin); - part.ri.reserve(end - begin); - body(begin, end, part.li, part.ri); - }); - batch.wait(); - } - std::size_t total = 0; - for (const auto& part : probe_parts_) { - total += part.li.size(); - } - li.resize(total); - ri.resize(total); - // The concat is the price the fan-out pays that the serial probe does - // not, and on a high-match join it is the whole regression: every row - // matching means `total == n`, i.e. two full index arrays copied - // again. Each part's destination slice is disjoint and known, so the - // copies go back to the workers; below the threshold the batch costs - // more than the memcpy it spreads. - constexpr std::size_t kMinParallelConcatRows = 1U << 16U; - const auto copy_part = [&](std::size_t w, std::size_t at) { - const auto& part = probe_parts_[w]; - std::ranges::copy(part.li, li.begin() + static_cast(at)); - std::ranges::copy(part.ri, ri.begin() + static_cast(at)); - }; - part_offsets_.resize(workers); - std::size_t at = 0; - for (std::size_t w = 0; w < workers; ++w) { - part_offsets_[w] = at; - at += probe_parts_[w].li.size(); - } - if (total >= kMinParallelConcatRows) { - auto batch = - pool.submit(workers, [&](std::size_t w) { copy_part(w, part_offsets_[w]); }); - batch.wait(); - } else { - for (std::size_t w = 0; w < workers; ++w) { - copy_part(w, part_offsets_[w]); - } - } - if (exec_->parallel_stats != nullptr) { - exec_->parallel_stats->parallel_probes.fetch_add(1, std::memory_order_relaxed); - } - return true; - } - - /// Swapped-mode probe: phase 1 walks right rows `head_of` resolves against - /// the left index, phase 2 expands the recorded chains into (li, ri). - /// The parallel path fans phase 1 out over contiguous right-row ranges and - /// phase 2 out over the per-range hit lists — each part's output slice - /// starts at the prefix sum of the parts before it, so workers write - /// disjoint slices and the result is byte-identical to the serial replay - /// (parts are visited in range order, ranges in row order). - template - void probe_swapped(std::size_t n_right, const HeadOf& head_of, std::vector& li, - std::vector& ri) { - const auto scan = [&](std::size_t begin, std::size_t end, std::vector& hits, - std::size_t& total) { - for (std::size_t r = begin; r < end; ++r) { - if (probe_is_null(r)) { - continue; - } - const std::size_t head = head_of(r); - if (head == kNil) { - continue; - } - hits.push_back(SwappedHit{.rrow = r, .head = head}); - for (std::size_t cur = head; cur != kNil; cur = index().chain_next[cur]) { - ++total; - } - } - }; - const auto replay = [&](const std::vector& hits, std::size_t pos) { - for (const SwappedHit& hit : hits) { - for (std::size_t cur = hit.head; cur != kNil; cur = index().chain_next[cur]) { - li[pos] = cur; - ri[pos] = hit.rrow; - ++pos; - } - } - }; - - const std::size_t workers = probe_parallel_workers(n_right); - if (workers == 0) { - std::vector hits; - std::size_t total = 0; - scan(0, n_right, hits, total); - li.assign(total, 0); - ri.assign(total, 0); - replay(hits, 0); - return; - } - auto& pool = process_worker_pool(); - const std::size_t grain = (n_right + workers - 1) / workers; - swapped_parts_.resize(workers); - { - auto batch = pool.submit(workers, [&](std::size_t w) { - auto& part = swapped_parts_[w]; - part.hits.clear(); - part.total = 0; - const std::size_t begin = w * grain; - const std::size_t end = std::min(n_right, begin + grain); - if (begin < end) { - scan(begin, end, part.hits, part.total); - } - }); - batch.wait(); - } - part_offsets_.resize(workers); - std::size_t total = 0; - for (std::size_t w = 0; w < workers; ++w) { - part_offsets_[w] = total; - total += swapped_parts_[w].total; - } - li.assign(total, 0); - ri.assign(total, 0); - { - auto batch = pool.submit( - workers, [&](std::size_t w) { replay(swapped_parts_[w].hits, part_offsets_[w]); }); - batch.wait(); - } - if (exec_->parallel_stats != nullptr) { - exec_->parallel_stats->parallel_probes.fetch_add(1, std::memory_order_relaxed); - } - } - - // Stream mode: walk the probe side (a left chunk), for each row look - // up the right-keyed chain and append (li, ri) in probe-scan order. - // Returns true if every probe row matched exactly once (li == 0..n-1). - // Only possible when the build side was unique; otherwise falls back - // to the chained walk. - template - auto probe_scalar(const Map& heads, std::size_t n, GetKey get, std::vector& li, - std::vector& ri) -> bool { - // One body for both paths, so the parallel and serial results cannot - // drift: the parallel one runs it per range, the serial one once. - const auto scan = [&](std::size_t begin, std::size_t end, std::vector& out_l, - std::vector& out_r) { - for (std::size_t l = begin; l < end; ++l) { - if (probe_is_null(l)) { - continue; - } - const std::size_t head = heads.find_head(get(l)); - if (head == kNil) { - continue; - } - for (std::size_t cur = head; cur != kNil; cur = index().chain_next[cur]) { - out_l.push_back(l); - out_r.push_back(cur); - } - } - }; - if (probe_ranges_parallel(n, li, ri, scan)) { - // `li_identity` means li == 0..n-1, which for a unique build side - // is exactly "every row matched" — the same test the serial path - // makes, just recovered from the totals. - return index().unique && li.size() == n; - } - if (index().unique) { - li.resize(n); - ri.resize(n); - std::size_t* lp = li.data(); - std::size_t* rp = ri.data(); - std::size_t out = 0; - for (std::size_t l = 0; l < n; ++l) { - if (probe_is_null(l)) { - continue; - } - const std::size_t head = heads.find_head(get(l)); - if (head == kNil) { - continue; - } - lp[out] = l; - rp[out] = head; - ++out; - } - li.resize(out); - ri.resize(out); - return out == n; - } - for (std::size_t l = 0; l < n; ++l) { - if (probe_is_null(l)) { - continue; - } - const std::size_t head = heads.find_head(get(l)); - if (head == kNil) { - continue; - } - std::size_t cur = head; - while (cur != kNil) { - li.push_back(l); - ri.push_back(cur); - cur = index().chain_next[cur]; - } - } - return false; - } - - // Probe with the build-side chain head already resolved per row. Same two - // shapes as `probe_scalar`, but the caller supplies the head instead of a - // key to hash — see `resolve_categorical_heads`. - template - auto probe_resolved(std::size_t n, GetHead head_of, std::vector& li, - std::vector& ri) -> bool { - const auto scan = [&](std::size_t begin, std::size_t end, std::vector& out_l, - std::vector& out_r) { - for (std::size_t l = begin; l < end; ++l) { - if (probe_is_null(l)) { - continue; - } - for (std::size_t cur = head_of(l); cur != kNil; cur = index().chain_next[cur]) { - out_l.push_back(l); - out_r.push_back(cur); - } - } - }; - if (probe_ranges_parallel(n, li, ri, scan)) { - return index().unique && li.size() == n; - } - if (index().unique) { - li.resize(n); - ri.resize(n); - std::size_t* lp = li.data(); - std::size_t* rp = ri.data(); - std::size_t out = 0; - for (std::size_t l = 0; l < n; ++l) { - if (probe_is_null(l)) { - continue; - } - const std::size_t head = head_of(l); - if (head == kNil) { - continue; - } - lp[out] = l; - rp[out] = head; - ++out; - } - li.resize(out); - ri.resize(out); - return out == n; - } - for (std::size_t l = 0; l < n; ++l) { - if (probe_is_null(l)) { - continue; - } - std::size_t cur = head_of(l); - while (cur != kNil) { - li.push_back(l); - ri.push_back(cur); - cur = index().chain_next[cur]; - } - } - return false; - } - - // A Categorical probe column is a dictionary plus one code per row, so - // every row sharing a code resolves to the same build chain. Resolve the - // DICTIONARY against the build index once — 252 entries for the symbol - // join — instead of rebuilding a string_view and hashing plus memcmp'ing - // it per row across 8M rows. That lookup was the single largest cost in - // the join profile (robin_hood string probe + __memcmp_avx2 + _Hash_bytes - // together ~57%). - // - // Rebuilt per chunk rather than cached on the operator: chunks of one scan - // usually share a dictionary, but nothing in the type guarantees it, and a - // stale memo would silently join against the wrong rows. |dict| lookups - // per chunk is noise next to |chunk rows|. - void resolve_categorical_heads(const std::vector& dict) { - probe_code_heads_.assign(dict.size(), kNil); - for (std::size_t c = 0; c < dict.size(); ++c) { - probe_code_heads_[c] = index().string_heads.find_head(std::string_view{dict[c]}); - } - } - - auto probe_chunk_against_right(Table left_chunk) -> std::expected { - if (pair_mode_) { - return probe_chunk_pair(std::move(left_chunk)); - } - const ColumnValue* key = left_chunk.find(keys_->front().left); - if (key == nullptr) { - return std::unexpected("join key not found in left chunk: " + keys_->front().left); - } - const auto* probe_entry = left_chunk.find_entry(keys_->front().left); - probe_validity_ = probe_entry != nullptr && probe_entry->validity.has_value() - ? &*probe_entry->validity - : nullptr; - - std::vector li; - std::vector ri; - const std::size_t n = left_chunk.rows(); - li.reserve(n); - ri.reserve(n); - bool li_identity = false; - - if (index().key_kind == ExprType::Int) { - const auto* col = std::get_if>(key); - if (col == nullptr) { - return std::unexpected("inner join: left key type mismatch"); - } - const auto* data = col->data(); - li_identity = - probe_scalar(index().i64_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); - } else if (index().key_kind == ExprType::Double) { - const auto* col = std::get_if>(key); - if (col == nullptr) { - return std::unexpected("inner join: left key type mismatch"); - } - const auto* data = col->data(); - li_identity = - probe_scalar(index().f64_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); - } else if (index().key_kind == ExprType::Bool) { - const auto* col = std::get_if>(key); - if (col == nullptr) { - return std::unexpected("inner join: left key type mismatch"); - } - li_identity = probe_scalar( - index().bool_heads, n, [&](std::size_t i) { return (*col)[i]; }, li, ri); - } else if (index().key_kind == ExprType::Date) { - const auto* col = std::get_if>(key); - if (col == nullptr) { - return std::unexpected("inner join: left key type mismatch"); - } - const auto* data = col->data(); - li_identity = - probe_scalar(index().date_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); - } else if (index().key_kind == ExprType::Timestamp) { - const auto* col = std::get_if>(key); - if (col == nullptr) { - return std::unexpected("inner join: left key type mismatch"); - } - const auto* data = col->data(); - li_identity = - probe_scalar(index().ts_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); - } else if (index().key_kind == ExprType::String) { - if (const auto* c_cat = std::get_if>(key)) { - const auto& dict = c_cat->dictionary(); - // Resolving the dictionary costs |dict| hash lookups and saves - // one per row, so it pays exactly when the dictionary is - // smaller than the chunk. A dictionary larger than the chunk - // (a narrow slice of a high-cardinality column) would hash more - // keys than there are rows to answer. - if (dict.size() < n) { - resolve_categorical_heads(dict); - li_identity = probe_resolved( - n, - [&](std::size_t i) { - return probe_code_heads_[static_cast(c_cat->code_at(i))]; - }, - li, ri); - } else { - li_identity = probe_scalar( - index().string_heads, n, - [&](std::size_t i) { - return std::string_view{ - dict[static_cast(c_cat->code_at(i))]}; - }, - li, ri); - } - } else if (const auto* c_str = std::get_if>(key)) { - li_identity = probe_scalar( - index().string_heads, n, [&](std::size_t i) { return (*c_str)[i]; }, li, ri); - } else { - return std::unexpected("inner join: left key type mismatch"); - } - } - - const std::size_t total = li_identity ? ri.size() : li.size(); - return assemble_output(std::move(left_chunk), li.data(), ri.data(), total, li_identity); - } - - auto probe_chunk_pair(Table left_chunk) -> std::expected { - const ir::JoinKey& k0 = keys_->at(0); - const ir::JoinKey& k1 = keys_->at(1); - const ColumnValue* key0 = left_chunk.find(k0.left); - if (key0 == nullptr) { - return std::unexpected("join key not found in left chunk: " + k0.left); - } - const ColumnValue* key1 = left_chunk.find(k1.left); - if (key1 == nullptr) { - return std::unexpected("join key not found in left chunk: " + k1.left); - } - const auto* col0 = std::get_if>(key0); - const auto* col1 = std::get_if>(key1); - if (col0 == nullptr || col1 == nullptr) { - return std::unexpected( - "inner join: left key type mismatch (two-key join expects " - "Int64)"); - } - const auto* e0 = left_chunk.find_entry(k0.left); - const auto* e1 = left_chunk.find_entry(k1.left); - const ValidityBitmap* v0 = - e0 != nullptr && e0->validity.has_value() ? &*e0->validity : nullptr; - const ValidityBitmap* v1 = - e1 != nullptr && e1->validity.has_value() ? &*e1->validity : nullptr; - - std::vector li; - std::vector ri; - const std::size_t n = left_chunk.rows(); - li.reserve(n); - ri.reserve(n); - - const auto* d0 = col0->data(); - const auto* d1 = col1->data(); - const auto is_null = [&](std::size_t r) { - return (v0 != nullptr && !(*v0)[r]) || (v1 != nullptr && !(*v1)[r]); - }; - const auto get_key = [&](std::size_t r) { - return JoinHashIndex::PairKey{.a = static_cast(d0[r]), - .b = static_cast(d1[r])}; - }; - const bool li_identity = probe_pair(n, is_null, get_key, li, ri); - - const std::size_t total = li_identity ? ri.size() : li.size(); - return assemble_output(std::move(left_chunk), li.data(), ri.data(), total, li_identity); - } - - // Same two shapes as `probe_scalar` (parallel fan-out via - // `probe_ranges_parallel`, then a unique-build fast path, then the - // general chained walk) but with an explicit null check instead of the - // single-bitmap `probe_is_null` member, since a probe row here is null - // when EITHER key is. - template - auto probe_pair(std::size_t n, IsNull is_null, GetKey get_key, std::vector& li, - std::vector& ri) -> bool { - const auto scan = [&](std::size_t begin, std::size_t end, std::vector& out_l, - std::vector& out_r) { - for (std::size_t l = begin; l < end; ++l) { - if (is_null(l)) { - continue; - } - const std::size_t head = index().pair_heads.find_head(get_key(l)); - if (head == kNil) { - continue; - } - for (std::size_t cur = head; cur != kNil; cur = index().chain_next[cur]) { - out_l.push_back(l); - out_r.push_back(cur); - } - } - }; - if (probe_ranges_parallel(n, li, ri, scan)) { - return index().unique && li.size() == n; - } - if (index().unique) { - li.resize(n); - ri.resize(n); - std::size_t* lp = li.data(); - std::size_t* rp = ri.data(); - std::size_t out = 0; - for (std::size_t l = 0; l < n; ++l) { - if (is_null(l)) { - continue; - } - const std::size_t head = index().pair_heads.find_head(get_key(l)); - if (head == kNil) { - continue; - } - lp[out] = l; - rp[out] = head; - ++out; - } - li.resize(out); - ri.resize(out); - return out == n; - } - for (std::size_t l = 0; l < n; ++l) { - if (is_null(l)) { - continue; - } - const std::size_t head = index().pair_heads.find_head(get_key(l)); - if (head == kNil) { - continue; - } - std::size_t cur = head; - while (cur != kNil) { - li.push_back(l); - ri.push_back(cur); - cur = index().chain_next[cur]; - } - } - return false; - } - - // Swapped mode: the hash index is on the left table, so the right table - // is the probe side, and output must still come out in left-row order. - // - // Phase 1 probes each right row once and remembers the head of every left - // chain it hit; phase 2 replays just those hits to fill (li, ri). The hash - // table is therefore probed once per right row for the whole join, not - // once per phase: a selective join over a large right side (q03 probes - // 3.2M lineitems to emit ~30K rows) no longer pays for 3.2M redundant - // cache-missing lookups. `hits` costs one entry per *matching* right row, - // so it is bounded by the output row count. - auto emit_swapped(const Table& left_table) -> std::expected { - if (pair_mode_) { - return emit_swapped_pair(left_table); - } - const ColumnValue* rkey = right_->find(keys_->front().right); - if (rkey == nullptr) { - return std::unexpected("join key not found in right table: " + keys_->front().right); - } - const std::size_t n_right = right_->rows(); - - // In swapped mode the index is on the left, so the right table is the - // probe side. Its null-keyed rows match nothing (see build_join_hash_index). - const auto* right_entry = right_->find_entry(keys_->front().right); - probe_validity_ = right_entry != nullptr && right_entry->validity.has_value() - ? &*right_entry->validity - : nullptr; - - std::vector li; - std::vector ri; - - // Every key kind reduces to "resolve right row r to a left chain head - // or kNil"; the map branches wrap the hash lookup, the categorical - // fast path hands the pre-resolved head straight through. One shape - // means `probe_swapped` is the single scan/replay implementation for - // both the serial and the parallel path. - auto do_phase1 = [&](auto&& key_at, const auto& heads) { - probe_swapped( - n_right, [&](std::size_t r) { return heads.find_head(key_at(r)); }, li, ri); - }; - // Same shape with the chain head already resolved — see - // `resolve_categorical_heads`. - auto do_phase1_resolved = [&](auto&& head_at) { probe_swapped(n_right, head_at, li, ri); }; - - if (index().key_kind == ExprType::Int) { - const auto* col = std::get_if>(rkey); - if (col == nullptr) - return std::unexpected("inner join: right key type mismatch"); - const auto* data = col->data(); - do_phase1([&](std::size_t r) { return data[r]; }, index().i64_heads); - } else if (index().key_kind == ExprType::Double) { - const auto* col = std::get_if>(rkey); - if (col == nullptr) - return std::unexpected("inner join: right key type mismatch"); - const auto* data = col->data(); - do_phase1([&](std::size_t r) { return data[r]; }, index().f64_heads); - } else if (index().key_kind == ExprType::Bool) { - const auto* col = std::get_if>(rkey); - if (col == nullptr) - return std::unexpected("inner join: right key type mismatch"); - do_phase1([&](std::size_t r) { return (*col)[r]; }, index().bool_heads); - } else if (index().key_kind == ExprType::Date) { - const auto* col = std::get_if>(rkey); - if (col == nullptr) - return std::unexpected("inner join: right key type mismatch"); - const auto* data = col->data(); - do_phase1([&](std::size_t r) { return data[r]; }, index().date_heads); - } else if (index().key_kind == ExprType::Timestamp) { - const auto* col = std::get_if>(rkey); - if (col == nullptr) - return std::unexpected("inner join: right key type mismatch"); - const auto* data = col->data(); - do_phase1([&](std::size_t r) { return data[r]; }, index().ts_heads); - } else if (index().key_kind == ExprType::String) { - if (const auto* c_cat = std::get_if>(rkey)) { - const auto& dict = c_cat->dictionary(); - if (dict.size() < n_right) { - resolve_categorical_heads(dict); - do_phase1_resolved([&](std::size_t r) { - return probe_code_heads_[static_cast(c_cat->code_at(r))]; - }); - } else { - do_phase1( - [&](std::size_t r) { - return std::string_view{ - dict[static_cast(c_cat->code_at(r))]}; - }, - index().string_heads); - } - } else if (const auto* c_str = std::get_if>(rkey)) { - do_phase1([&](std::size_t r) { return (*c_str)[r]; }, index().string_heads); - } else { - return std::unexpected("inner join: right key type mismatch"); - } - } - - // Output order is the order phase 1 visited the hits — right-scan - // (probe) order. Row order is outside the join contract (SPEC.md - // §5.6), so there's no correctness reason to reassemble by left row - // instead; doing so was actively harmful, permuting the output away - // from the probe side's natural scan order and hurting cache locality - // on any downstream join that probes this join's output. - Table left_copy; - left_copy.columns.reserve(left_table.columns.size()); - for (const auto& c : left_table.columns) { - left_copy.add_column(c.name, *c.column); - left_copy.columns.back().validity = c.validity; - } - return assemble_output(std::move(left_copy), li.data(), ri.data(), li.size()); - } - - // Swapped pair-mode counterpart of `emit_swapped`: the pair index is on - // the build-side left table, so the right table's rows are the probe - // side. Reuses - // `probe_swapped` unchanged -- it is already generic over a - // `head_of(row)` callback -- with the null check folded into `head_of` - // itself (returning `kNil`) instead of the single-bitmap `probe_is_null` - // member, since a row here is null when EITHER key is. - auto emit_swapped_pair(const Table& left_table) -> std::expected { - const ir::JoinKey& k0 = keys_->at(0); - const ir::JoinKey& k1 = keys_->at(1); - const ColumnValue* rkey0 = right_->find(k0.right); - if (rkey0 == nullptr) { - return std::unexpected("join key not found in right table: " + k0.right); - } - const ColumnValue* rkey1 = right_->find(k1.right); - if (rkey1 == nullptr) { - return std::unexpected("join key not found in right table: " + k1.right); - } - const auto* col0 = std::get_if>(rkey0); - const auto* col1 = std::get_if>(rkey1); - if (col0 == nullptr || col1 == nullptr) { - return std::unexpected( - "inner join: right key type mismatch (two-key join expects Int64)"); - } - const auto* e0 = right_->find_entry(k0.right); - const auto* e1 = right_->find_entry(k1.right); - const ValidityBitmap* v0 = - e0 != nullptr && e0->validity.has_value() ? &*e0->validity : nullptr; - const ValidityBitmap* v1 = - e1 != nullptr && e1->validity.has_value() ? &*e1->validity : nullptr; - const auto* d0 = col0->data(); - const auto* d1 = col1->data(); - const std::size_t n_right = right_->rows(); - - const auto head_of = [&](std::size_t r) -> std::size_t { - if ((v0 != nullptr && !(*v0)[r]) || (v1 != nullptr && !(*v1)[r])) { - return kNil; - } - const JoinHashIndex::PairKey key{.a = static_cast(d0[r]), - .b = static_cast(d1[r])}; - return index().pair_heads.find_head(key); - }; - - std::vector li; - std::vector ri; - probe_swapped(n_right, head_of, li, ri); - - Table left_copy; - left_copy.columns.reserve(left_table.columns.size()); - for (const auto& c : left_table.columns) { - left_copy.add_column(c.name, *c.column); - left_copy.columns.back().validity = c.validity; - } - return assemble_output(std::move(left_copy), li.data(), ri.data(), li.size()); - } - - auto assemble_output(Table left_side, const std::size_t* li, const std::size_t* ri, - std::size_t total, bool li_identity = false, bool ri_identity = false) - -> std::expected { - Table output; - if (!right_emit_ready_) { - if (auto ready = setup_right_emit_schema(left_side); !ready.has_value()) { - return std::unexpected(std::move(ready.error())); - } - } - output.columns.reserve(left_side.columns.size() + right_emit_idx_.size()); - - // A stream with no matches still has a schema. Returning a bare empty - // table here used to be harmless only because the regular chunked path - // normally has another node to provide one; the whole-table adapter - // must be equivalent to join_table_impl even for an empty result. - if (total == 0) { - for (std::size_t i = 0; i < left_side.columns.size(); ++i) { - output.add_column(std::string(left_emit_names_[i]), - make_empty_like(*left_side.columns[i].column)); - } - for (std::size_t e = 0; e < right_emit_idx_.size(); ++e) { - output.add_column(std::string(right_emit_names_[e]), - make_empty_like(*right_->columns[right_emit_idx_[e]].column)); - } - return output; - } - - // Gather a batch of columns in ONE fan-out. Calling `gather_column` per - // column instead lets each call fan out its own rows, which submits and - // waits a batch PER COLUMN — see `gather_columns_batched` for the - // measurement that ruled that out. This is an inner join, so no index - // carries a `kNull` sentinel and no job needs `indivisible`. - const auto gather_batch = - [&](std::span jobs) -> std::vector { - return gather_columns_batched(jobs, total, exec_, [&](std::size_t j) -> GatheredColumn { - const auto& job = jobs[j]; - ColumnValue gathered = gather_column(*job.column, job.idx, total, nullptr); - std::optional val; - if (job.validity != nullptr) { - ValidityBitmap dst(total, false); - gather_validity_range(dst, *job.validity, - std::span{job.idx, total}, 0, total); - val = std::move(dst); - } - return {std::move(gathered), std::move(val)}; - }); - }; - - // li_identity: every probe row matched exactly once, so left columns - // can be passed through directly (shared_ptr share) instead of - // gathered. Do NOT move the underlying ColumnValue — the shared_ptr - // may be aliased by upstream state (e.g., re-runnable source). - const auto left_name = [&](std::size_t i, const ColumnEntry& lc) -> const std::string& { - return i < left_emit_names_.size() ? left_emit_names_[i] : lc.name; - }; - - // The left's ordering, restated in the output's names, when this batch - // emitted the left rows in their own order. Same rule and reasoning as - // the materialized join in join.cpp -- a join promises no order, but a - // path that produces one should say so, and the claim is proved from - // the emitted index array rather than from which mode ran. Computed - // before the identity branch below, which renames left columns in place. - const auto carried_ordering = [&]() -> std::vector { - if (!left_side.properties().ordering().has_value()) { - return {}; - } - if (!li_identity) { - for (std::size_t i = 1; i < total; ++i) { - if (li[i] < li[i - 1]) { - return {}; - } - } - } - std::vector out; - for (const auto& key : *left_side.properties().ordering()) { - std::optional emitted; - for (std::size_t i = 0; i < left_side.columns.size(); ++i) { - if (left_side.columns[i].name == key.name) { - emitted = left_name(i, left_side.columns[i]); - break; - } - } - if (!emitted.has_value()) { - return {}; // a key the output cannot name - } - out.push_back( - ir::OrderKey{.name = std::move(*emitted), .ascending = key.ascending}); - } - return out; - }(); - if (li_identity && total == left_side.rows()) { - for (std::size_t i = 0; i < left_side.columns.size(); ++i) { - auto& lc = left_side.columns[i]; - std::string name = left_name(i, lc); - lc.name = name; - output.index[std::move(name)] = output.columns.size(); - output.columns.push_back(std::move(lc)); - } - } else { - std::vector jobs; - jobs.reserve(left_side.columns.size()); - for (const auto& lc : left_side.columns) { - jobs.push_back({.column = lc.column.get(), - .validity = lc.validity.has_value() ? &*lc.validity : nullptr, - .idx = li, - .indivisible = false}); - } - auto gathered = gather_batch(jobs); - for (std::size_t i = 0; i < left_side.columns.size(); ++i) { - const auto& lc = left_side.columns[i]; - if (gathered[i].second.has_value()) { - output.add_column(left_name(i, lc), std::move(gathered[i].first), - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - std::move(*gathered[i].second)); - } else { - output.add_column(left_name(i, lc), std::move(gathered[i].first)); - } - } - } - - // ri_identity: every emitted row consumes the next probe-side row - // exactly once (two-phase deferred probe with a unique build side), - // so probe columns are shared rather than gathered — the same - // reasoning as li_identity above. - const bool share_right = ri_identity && total == right_->rows(); - if (share_right) { - for (std::size_t e = 0; e < right_emit_idx_.size(); ++e) { - output.add_column_from(std::string(right_emit_names_[e]), - right_->columns[right_emit_idx_[e]]); - } - } else { - std::vector jobs; - jobs.reserve(right_emit_idx_.size()); - for (const auto index : right_emit_idx_) { - const auto& rc = right_->columns[index]; - jobs.push_back({.column = rc.column.get(), - .validity = rc.validity.has_value() ? &*rc.validity : nullptr, - .idx = ri, - .indivisible = false}); - } - auto gathered = gather_batch(jobs); - for (std::size_t e = 0; e < right_emit_idx_.size(); ++e) { - std::string name = right_emit_names_[e]; - if (gathered[e].second.has_value()) { - output.add_column(std::move(name), std::move(gathered[e].first), - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - std::move(*gathered[e].second)); - } else { - output.add_column(std::move(name), std::move(gathered[e].first)); - } - } - } - if (!carried_ordering.empty()) { - output.set_properties(output.properties().with_ordering(carried_ordering)); - } - return output; - } -}; - -/// The streaming probe as an operator: a source of probe-side chunks, one -/// completed build, and nothing else. -/// -/// This is Phase 4's `HashProbe`. It owns no build -- it reads one through -/// `JoinProbe`'s `shared_ptr` handles -- and it does not know which -/// side of the join was hashed, because by the time it exists that is settled. -/// Two things follow, and they are the reason it is a type rather than a loop -/// inside the join: it can be constructed next to a build it did not run, and -/// several of it can read one build at once, which is what a per-worker morsel -/// chain needs. -/// -/// The empty-schema carrier travels with it, because "this join produced no -/// rows but still has a schema" is a property of probing, not of the operator -/// that decided the orientation. -class JoinProbeOperator final : public Operator { - public: - /// `preserve_empty_morsels` is the same contract every map kernel in a - /// morsel chain honours: one input morsel yields exactly one identified - /// output morsel, because the ordered ring indexes by sequence and a - /// coalesced empty result would be a lost slot rather than a smaller - /// answer. A probe needs it more than a filter does -- a morsel of - /// probe-side rows that matches nothing is entirely ordinary. - JoinProbeOperator(OperatorPtr child, JoinProbe probe, bool preserve_empty_morsels = false) - : child_(std::move(child)), - probe_(std::move(probe)), - preserve_empty_(preserve_empty_morsels) {} - - [[nodiscard]] auto next() -> std::expected, std::string> override { - while (true) { - auto chunk_res = child_->next(); - if (!chunk_res.has_value()) { - return std::unexpected(std::move(chunk_res.error())); - } - if (!chunk_res.value().has_value()) { - if (!emitted_nonempty_ && empty_schema_.has_value()) { - auto schema = std::move(*empty_schema_); - empty_schema_.reset(); - return std::optional{table_to_chunk(std::move(schema))}; - } - return std::optional{}; - } - Chunk input = std::move(*chunk_res.value()); - // A morsel's identity travels with it. `sequence` and `row_offset` - // identify which morsel this is, not which rows it holds, so a - // probe propagates them unchanged exactly as a filter does -- - // both change the row count, and neither changes which morsel it - // is answering for. The ordered ring rejects a chunk that arrives - // without them. - const std::uint64_t sequence = input.sequence; - const std::size_t row_offset = input.row_offset; - auto out = probe_.probe_chunk_against_right(chunk_to_table(std::move(input))); - if (!out.has_value()) { - return std::unexpected(std::move(out.error())); - } - if (out->rows() == 0 && !preserve_empty_) { - // Keep the planned empty table as a schema carrier. A join - // with no matches still has its left and right output columns; - // without this, a materializing sink sees no chunks at all. - empty_schema_ = std::move(*out); - continue; - } - emitted_nonempty_ = true; - Chunk result = table_to_chunk(std::move(*out)); - result.sequence = sequence; - result.row_offset = row_offset; - return std::optional{std::move(result)}; - } - } - - private: - OperatorPtr child_; - JoinProbe probe_; - bool preserve_empty_ = false; - std::optional
empty_schema_; - bool emitted_nonempty_ = false; -}; - -/// How many workers a probe over an already-materialized probe side may fan -/// out to, or 0 to decline. Defined with the morsel machinery below. -[[nodiscard]] auto probe_morsel_workers(const Table& input, const ExecutionContext& exec) - -> std::size_t; - -/// Run a probe over the morsels of an already-materialized probe side: one -/// `JoinProbeOperator` per worker, all reading the same build, results merged -/// in morsel order. Defined below, next to the morsel executor it uses. -[[nodiscard]] auto build_probe_morsel_pipeline(Table input, const JoinProbe& probe, - std::size_t workers, const ExecutionContext& exec) - -> std::expected; - -/// Inner hash join for single-key no-predicate joins. -/// -/// Two execution modes: -/// - Stream: right is small (<= kStreamRightThreshold). Build a chained -/// hash index on the materialized right, then probe each left chunk -/// streamed from the child. Matches the classic star-join shape. -/// - Swapped: right is large and n_left < n_right. Materialize left, -/// build the hash index on left, iterate right rows once and emit output -/// in that same right-scan (probe) order (baseline's -/// `build_indices_from_right_scan` equivalent) — row order is outside the -/// join contract (SPEC.md §5.6), and preserving the probe side's scan order -/// instead of reassembling by left row keeps cache locality for any -/// downstream join that probes this join's output. Much better cache -/// behavior overall when the smaller side fits. -/// -/// Name conflicts are resolved with the same `_right` suffix rule as -/// `join_table_impl`. -class ChunkedInnerJoinOperator final : public Operator { - public: - ChunkedInnerJoinOperator(OperatorPtr left, Table right, const std::vector* keys, - const ExecutionContext& exec, ir::JoinSuffixPolicy suffix = {}, - const std::vector* pending_order = nullptr, - physical::JoinParallelism par = {}) - : left_(std::move(left)), - right_(std::make_shared
(std::move(right))), - keys_(keys), - par_(par), - pending_order_(pending_order) { - bind_probe(keys, std::move(suffix), exec); - } - - /// Deferred-probe variant: the right side is an undecoded lazy scan (plus - /// its Project/Rename wrappers), interpreted only after this join has - /// published build-side key bounds into the scan's filter slot. The - /// registry/scalars/externs pointers are the interpret context and outlive - /// the operator. - ChunkedInnerJoinOperator(OperatorPtr left, const ir::Node* right_node, - const TableRegistry* registry, const ScalarRegistry* scalars, - const ExternRegistry* externs, const ExecutionContext& exec, - const std::vector* keys, const DeferredScan* probe, - std::string probe_name, ir::JoinSuffixPolicy suffix = {}, - const std::vector* pending_order = nullptr, - physical::JoinParallelism par = {}) - : left_(std::move(left)), - keys_(keys), - deferred_probe_(probe), - deferred_probe_name_(std::move(probe_name)), - deferred_right_node_(right_node), - deferred_registry_(registry), - deferred_scalars_(scalars), - deferred_externs_(externs), - deferred_exec_(&exec), - par_(par), - pending_order_(pending_order) { - bind_probe(keys, std::move(suffix), exec); - } - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (auto err = run_build()) { - return std::unexpected(std::move(*err)); - } - - if (mode_ == Mode::Precomputed) { - if (swapped_emitted_) { - return std::optional{}; - } - swapped_emitted_ = true; - if (precomputed_output_.rows() == 0) { - return std::optional{}; - } - return std::optional{table_to_chunk(std::move(precomputed_output_))}; - } - - if (mode_ == Mode::Swapped) { - if (swapped_emitted_) { - return std::optional{}; - } - swapped_emitted_ = true; - if (!left_table_.has_value()) { - return std::unexpected( - "ChunkedInnerJoinOperator: swapped mode without a materialized left table"); - } - auto out = probe_.emit_swapped(*left_table_); - if (!out.has_value()) { - return std::unexpected(std::move(out.error())); - } - if (out->rows() == 0) { - return std::optional{}; - } - return std::optional{table_to_chunk(std::move(*out))}; - } - - // Stream mode is the probe, and the probe is its own operator. The - // join constructs it on first use and delegates from here on: - // everything left in this class is build-side. - if (auto err = ensure_probe_op()) { - return std::unexpected(std::move(*err)); - } - return probe_op_->next(); - } - - /// Run this join's build phase to completion. - /// - /// The build is a phase with an explicit caller now, not a side effect of - /// whoever happens to pull the first chunk. `build_physical_join` runs it - /// at plan-execution time; `next()` still calls it, because a join reached - /// by any other path must work and because idempotence is what makes both - /// callers safe. After it returns the index is immutable and the probe - /// side can stream through it -- that is the barrier, stated as a call - /// rather than as a comment about `initialized_`. - /// - /// Deliberately NOT what this changes: it does not overlap the build with - /// anything. Overlapping a join's two sides has been tried twice and - /// regressed both times (`32889afd`, `27cb4a27`); this makes the build - /// schedulable, and what to schedule it against stays an open, measured - /// question. - auto run_build() -> std::optional { - if (initialized_) { - return std::nullopt; - } - if (auto err = initialize()) { - return err; - } - initialized_ = true; - return std::nullopt; - } - - private: - enum class Mode : std::uint8_t { Stream, Swapped, Precomputed }; - - /// The chain terminator, shared with the index this operator probes. - static constexpr std::size_t kNil = kJoinNil; - - // Build-on-right is preferred when right is small enough that probing - // it from streaming left chunks is cache-friendly. Above this, we - // materialize left to pick the smaller build side. - static constexpr std::size_t kStreamRightThreshold = 65536; - - /// Hand the probe what it needs before anything runs, including joint - /// ownership of the right side. The `shared_ptr`'s identity is stable from - /// construction even where the table it points at is filled later (the - /// deferred path fills it once the scan resolves), so this binds once and - /// the build phase writes through it. - void bind_probe(const std::vector* keys, ir::JoinSuffixPolicy suffix, - const ExecutionContext& exec) { - probe_.keys_ = keys; - probe_.suffix_ = std::move(suffix); - probe_.exec_ = &exec; - probe_.right_ = right_; - probe_.probe_plan_ = par_.probe; - } - - auto initialize() -> std::optional { - if (keys_->size() == 2) { - return initialize_pair(); - } - if (keys_->size() != 1) { - return "ChunkedInnerJoinOperator only supports single-key or two-Int64-key joins"; - } - if (deferred_probe_ != nullptr) { - if (auto err = resolve_deferred_probe()) { - return err; - } - if (mode_ == Mode::Precomputed) { - return std::nullopt; - } - } - const std::string& left_key_name = keys_->front().left; - const std::string& right_key_name = keys_->front().right; - const ColumnValue* rkey = right_->find(right_key_name); - if (rkey == nullptr) { - return "join key not found in right table: " + right_key_name; - } - ExprType key_kind = ExprType::Int; - if (auto err = detect_join_key_kind(*rkey, key_kind)) { - return err; - } - - const std::size_t n_right = right_->rows(); - - // Small right: index it without ever measuring the left, which is the - // one orientation this join can choose without draining a child. - if (n_right <= kStreamRightThreshold) { - return adopt_build(build_join_side(*right_, right_key_name, key_kind, - JoinOrientation::BuildRight, - build_partitions(n_right))); - } - - Table left_table; - if (use_materialized_left_ && left_materialized_.has_value()) { - // The deferred-probe path already drained the left child. - left_table = std::move(*left_materialized_); - left_materialized_.reset(); - use_materialized_left_ = false; - } else { - auto left_res = MaterializeOperator(std::move(left_)).run(); - if (!left_res.has_value()) { - return std::move(left_res.error()); - } - left_table = std::move(*left_res); - } - const std::size_t n_left = left_table.rows(); - - // Evaluated only when swapping was otherwise preferred: it is not a - // pure predicate (it can set up the probe's right-emit schema), so - // asking it unconditionally would move work the short-circuit used to - // skip. - const bool order_pays = - n_left < n_right && order_preserving_pays(left_table, n_left, n_right); - auto outcome = choose_and_build_single_key(left_table, *right_, left_key_name, - right_key_name, key_kind, order_pays, - build_partitions(std::min(n_left, n_right))); - return adopt_build(std::move(outcome), std::move(left_table)); - } - - /// How many partitions this join's hash build may fill concurrently, or 1 - /// to build it serially. - /// - /// The floor (`1U << 17U`) and the worker cap (`min(compute_budget, pool, - /// 64)`) used to be computed here; they are now the `hash-build` phase of - /// the physical plan (`physical::join_hash_build_parallelism`), resolved by - /// the builder and read from `par_.build`. src/runtime/PARALLELISM.md, - /// "Target: parallelism as a plan decision". The conditions that stay here - /// are the ones only the operator can judge: the `IBEX_JOIN_BUILD_SERIAL` - /// kill switch, whether it is nested under another fan-out - /// (`on_worker_pool_thread` -- no nested pool submissions), and whether - /// *this* build side cleared the floor (its row count is not known until - /// the side is materialized). The floor is higher than the probe's because - /// a partitioned build makes three passes over the keys where a serial - /// build makes one. - [[nodiscard]] auto build_partitions(std::size_t n) const -> std::size_t { - // Kill switch, and the A/B handle: with it set, the same binary runs - // the serial build, so the two can be interleaved without rebuilding. - if (std::getenv("IBEX_JOIN_BUILD_SERIAL") != nullptr) { - return 1; - } - if (par_.build.decline != physical::FanOutDecline::None || par_.build.worker_cap < 2 || - on_worker_pool_thread() || n < par_.build.row_floor) { - return 1; - } - // Telemetry only (see `parallel_hash_builds`): the fan-out is byte-identical - // to the serial build, so a gate that silently stopped matching would lose - // the parallelism with every test still green. Counted here rather than at - // the four call sites, which all route the result straight into the build. - if (probe_.exec_ != nullptr && probe_.exec_->parallel_stats != nullptr) { - probe_.exec_->parallel_stats->parallel_hash_builds.fetch_add(1, - std::memory_order_relaxed); - } - return par_.build.worker_cap; - } - - /// Publish what a build produced, and hand back the orientation it chose. - /// The single place `probe_.index_` is written: `probe_.index()` gives the - /// probe a `const` reference, so the barrier stays a compile error to - /// cross rather than a convention. - auto publish_build(std::expected outcome) - -> std::expected { - if (!outcome.has_value()) { - return std::unexpected(std::move(outcome.error())); - } - probe_.index_ = std::move(outcome->index); - return outcome->orientation; - } - - /// Apply what a build phase decided: publish the index and put this - /// operator into the shape that orientation implies. The build returns a - /// value; every member write that follows from it happens here and - /// nowhere else. - /// - /// `left_table` is the materialized left when the decision needed one -- - /// it becomes either the scanned-once swapped side or the single chunk - /// the stream path replays, depending on which way the build went. - auto adopt_build(std::expected outcome, - std::optional
left_table = std::nullopt) -> std::optional { - auto orientation = publish_build(std::move(outcome)); - if (!orientation.has_value()) { - return std::move(orientation.error()); - } - if (*orientation == JoinOrientation::BuildLeft) { - left_table_ = std::move(left_table); - mode_ = Mode::Swapped; - return std::nullopt; - } - // BuildRight: the other side streams through the index, which is what - // `JoinProbeOperator` does. A left that has already been drained -- - // either by the orientation decision above, or earlier by the - // deferred-probe path publishing its key bounds -- is replayed as one - // chunk rather than re-wrapped in its original operator, exactly as - // `use_materialized_left_` used to do inline. - // - // The second case is not hypothetical and cost a segfault to find: - // the deferred path drains `left_` to publish a filter, and if the - // resolved right then lands under `kStreamRightThreshold` the fast - // path arrives here with no `left_table` and a moved-from `left_`. - std::optional
materialized_probe_side; - if (left_table.has_value()) { - materialized_probe_side = std::move(*left_table); - } else if (use_materialized_left_ && left_materialized_.has_value()) { - materialized_probe_side = std::move(*left_materialized_); - left_materialized_.reset(); - use_materialized_left_ = false; - } - - // The probe side is kept rather than consumed. Building the probe - // operator here would settle a question a caller above may want to - // answer differently: a map pipeline over this join can take the probe - // and run it at the head of its own worker chains, which is the Umbra - // shape -- one build pipeline, then a probe pipeline whose maps run in - // the same worker as the probe. `take_fusible_probe` is that handoff, - // and it has to happen before `ensure_probe_op` commits. - probe_side_ = std::move(materialized_probe_side); - if (!probe_side_.has_value() && left_ == nullptr) { - // Every way of losing the probe side is a bug in the branches - // above, and one of them was. Aborting with a name beats - // dereferencing null inside a pool thread, which is what the - // deferred fast-path case actually did. - invariant_violation("join probe: no probe-side source after the build phase"); - } - return std::nullopt; - } - - /// Build Stream mode's probe operator, once, on first use. - /// - /// Split from `adopt_build` so a caller that wants to fuse this probe into - /// its own pipeline has a window in which the decision is still open -- - /// see `take_fusible_probe`. - auto ensure_probe_op() -> std::optional { - if (probe_op_ != nullptr) { - return std::nullopt; - } - // A materialized probe side can be morselized: one probe per worker, - // each over its own morsel, all reading this one immutable build. - // Opt-in, and `probe_morsel_workers` records the measurement that - // explains why. - if (probe_side_.has_value() && probe_.exec_ != nullptr) { - if (const std::size_t workers = probe_morsel_workers(*probe_side_, *probe_.exec_); - workers >= 2) { - auto pipeline = build_probe_morsel_pipeline(std::move(*probe_side_), probe_, - workers, *probe_.exec_); - if (!pipeline.has_value()) { - return std::move(pipeline.error()); - } - probe_side_.reset(); - probe_op_ = std::move(*pipeline); - return std::nullopt; - } - } - OperatorPtr probe_source = - probe_side_.has_value() ? make_table_source(std::move(*probe_side_)) : std::move(left_); - probe_side_.reset(); - probe_op_ = std::make_unique(std::move(probe_source), std::move(probe_)); - return std::nullopt; - } - - public: - /// One build, one probe side, ready to be run by someone else. - struct FusibleProbe { - Table probe_side; - JoinProbe probe; - }; - - /// Hand the probe and its input to a caller that will run them itself. - /// Empty unless this join settled on `BuildRight` and nothing has pulled - /// from it yet: a swapped or precomputed join emits one table and has no - /// probe pipeline to give. - /// - /// A probe side that is still streaming gets materialized here, which - /// sounds like a cost added and is not: the caller is a parallel map - /// pipeline, which was going to materialize the join's OUTPUT and - /// morselize that. This materializes the probe side instead, and the - /// join's output is then never assembled at all -- it is produced a morsel - /// at a time inside the workers. Which of the two tables is larger is a - /// real question and a measured one; it is not a question of whether a - /// materialization exists. - /// - /// The operator is left without a probe and must be discarded. - [[nodiscard]] auto take_fusible_probe() - -> std::expected, std::string> { - if (mode_ != Mode::Stream || probe_op_ != nullptr) { - return std::optional{}; - } - if (!probe_side_.has_value()) { - if (left_ == nullptr) { - return std::optional{}; - } - auto drained = MaterializeOperator(std::move(left_)).run(); - if (!drained.has_value()) { - return std::unexpected(std::move(drained.error())); - } - probe_side_ = std::move(*drained); - } - FusibleProbe out{.probe_side = std::move(*probe_side_), .probe = std::move(probe_)}; - probe_side_.reset(); - return std::optional{std::move(out)}; - } - - private: - /// Two-fixed-width-int-key path: narrow first cut of the streaming - /// two-key join (plans/parallelism-overview.md's "stream multi-key - /// joins" item). Non-deferred case: `right_` is already a whole `Table` - /// by construction (the call site materializes it, same as the - /// single-key path), so the only real decision left is which side to - /// index: this materializes `left_` too and builds on whichever side is - /// smaller -- the same motivation as `initialize()`'s single-key swap - /// decision, and necessary here because the call site cannot know in - /// advance which side a two-key join chain puts on which name (TPC-H - /// q09's `lineitem` join has the multi-million-row side as `right_`; - /// indexing it unconditionally was measured a >2x regression before this - /// fix). Deferred case: see `resolve_deferred_probe_pair`. - auto initialize_pair() -> std::optional { - if (deferred_probe_ != nullptr) { - return resolve_deferred_probe_pair(); - } - const ir::JoinKey& k0 = keys_->at(0); - const ir::JoinKey& k1 = keys_->at(1); - // Checked before the left child is drained, so an unusable right key - // still costs nothing to report. - if (auto right_keys = pair_key_columns(*right_, k0.right, k1.right, "right"); - !right_keys.has_value()) { - return std::move(right_keys.error()); - } - Table left_table; - if (use_materialized_left_ && left_materialized_.has_value()) { - // `resolve_deferred_probe_pair` already drained the left child - // while deciding whether a scan filter was worth publishing. - left_table = std::move(*left_materialized_); - left_materialized_.reset(); - use_materialized_left_ = false; - } else { - auto left_res = MaterializeOperator(std::move(left_)).run(); - if (!left_res.has_value()) { - return std::move(left_res.error()); - } - left_table = std::move(*left_res); - } - probe_.pair_mode_ = true; - // On the BuildRight side of this decision the left is already fully - // materialized, so `adopt_build` drains it as a single chunk through - // the existing `use_materialized_left_` mechanism rather than - // re-wrapping it in an operator. - auto outcome = - choose_and_build_pair(left_table, *right_, k0, k1, - build_partitions(std::min(left_table.rows(), right_->rows()))); - return adopt_build(std::move(outcome), std::move(left_table)); - } - - /// Deferred-probe POC for the two-key path (plans/parallelism-overview.md - /// "deferred scan filtering for two-key joins", TPC-H q09's `lineitem` - /// join). Reuses the existing single-key deferred-scan machinery - /// unchanged: builds the (small) left side first, publishes a - /// `DynamicScanFilter` over `keys_->at(0)` ONLY -- one component, not - /// both -- into the scan's filter slot, then lets the source's normal - /// decode-time pruning narrow the right side before it is ever - /// materialized. Membership in one component is necessary but not - /// sufficient for the pair match, so this can only produce harmless - /// false positives (rows sharing q09's l_partkey but not l_suppkey); - /// `choose_and_build_pair`'s exact pair probe afterward is what - /// actually enforces both keys, unchanged from the non-deferred path. - /// - /// Same `kStreamRightThreshold` gate as the single-key - /// `resolve_deferred_probe`: below it, publishing a filter (a Bloom plus - /// a sort/unique pass over the whole build side) can only add cost, not - /// recover it, since the deferred side was never going to be expensive to - /// decode in the first place. First cut of this POC always materialized - /// left and published a filter regardless of size -- measured a clean, - /// unanimous +8.4% regression on q05 (its `join supplier on - /// {l_suppkey=s_suppkey, n_nationkey=s_nationkey}` is exactly this - /// shape, `supplier` fitting in one row group with nothing to prune). - /// Below the threshold this falls through to `initialize_pair`'s - /// ordinary side-picking, reusing the already-drained left side via - /// `left_materialized_`/`use_materialized_left_` rather than draining it - /// twice. - /// - /// No two-phase probe (`try_two_phase_probe`'s candidate-selection - /// optimization) here -- that is a further, separable lever on top of - /// scan-altitude pruning, and this POC is scoped to answering whether - /// pruning the scan itself is worth it at all before adding more on top. - auto resolve_deferred_probe_pair() -> std::optional { - DynamicScanFilter& slot = *deferred_probe_->filter; - if (deferred_probe_->lazy->rows() > kStreamRightThreshold) { - auto left_res = MaterializeOperator(std::move(left_)).run(); - if (!left_res.has_value()) { - return std::move(left_res.error()); - } - publish_build_filter_column(*left_res, keys_->at(0).left, slot); - left_materialized_ = std::move(*left_res); - use_materialized_left_ = true; - } - slot.ready = true; - auto right = interpret_node(*deferred_right_node_, *deferred_registry_, deferred_scalars_, - deferred_externs_, *deferred_exec_); - if (!right.has_value()) { - return std::move(right.error()); - } - *right_ = std::move(*right); - deferred_probe_ = nullptr; - if (std::getenv("IBEX_DEBUG_PAIR_DEFER") != nullptr) { - ibex::formatting::print(stderr, - "[pair_defer] filter_published={} right_rows_after_filter={}\n", - static_cast(use_materialized_left_), right_->rows()); - } - return initialize_pair(); - } - - /// The probe side is an undecoded lazy scan. When it is worth it, drain - /// the build (left) side first and publish its key filter (membership + - /// bounds) into the scan's filter slot, so the scan skips materializing - /// rows that cannot match. Every path marks the slot ready before the - /// scan is interpreted; the filter is an optimization the slot may - /// simply not carry. - auto resolve_deferred_probe() -> std::optional { - DynamicScanFilter& slot = *deferred_probe_->filter; - // Pre-filter row count: an upper bound on the decoded size, good - // enough to decide whether the probe side is large enough to bother. - if (deferred_probe_->lazy->rows() > kStreamRightThreshold) { - auto left_res = MaterializeOperator(std::move(left_)).run(); - if (!left_res.has_value()) { - return std::move(left_res.error()); - } - publish_build_filter(*left_res, slot); - left_materialized_ = std::move(*left_res); - use_materialized_left_ = true; - } - slot.ready = true; - if (use_materialized_left_) { - TwoPhase outcome = TwoPhase::NotApplicable; - if (auto err = try_two_phase_probe(slot, outcome)) { - return err; - } - if (outcome == TwoPhase::Precomputed) { - deferred_probe_ = nullptr; - mode_ = Mode::Precomputed; - return std::nullopt; - } - if (outcome == TwoPhase::RightMaterialized) { - // Phase A ran but full two-phase declined; its selection was - // reused to materialize right_, so fall through to the - // ordinary side-picking in initialize(). - deferred_probe_ = nullptr; - return std::nullopt; - } - } - auto right = interpret_node(*deferred_right_node_, *deferred_registry_, deferred_scalars_, - deferred_externs_, *deferred_exec_); - if (!right.has_value()) { - return std::move(right.error()); - } - *right_ = std::move(right.value()); - deferred_probe_ = nullptr; - return std::nullopt; - } - - enum class TwoPhase : std::uint8_t { NotApplicable, RightMaterialized, Precomputed }; - - /// Interpret the Project/Rename/Update wrappers over an already - /// materialized scan table by shadowing the scan name in a registry - /// copy — the Scan case hits the registry before the deferred fallback. - auto interpret_wrapped_right(Table scan_table) -> std::optional { - TableRegistry local = *deferred_registry_; - local.insert_or_assign(deferred_probe_name_, std::move(scan_table)); - auto right = interpret_node(*deferred_right_node_, local, deferred_scalars_, - deferred_externs_, *deferred_exec_); - if (!right.has_value()) { - return std::move(right.error()); - } - *right_ = std::move(right.value()); - return std::nullopt; - } - - /// Late materialization across the join (decode-fusion stage 5): probe - /// with just the scan's key column, then decode the payload columns only - /// for the rows that actually matched. When every survivor matched - /// exactly one build row (unique build keys — the common star shape), - /// the probe-side columns pass into the output without a gather. - /// - /// NotApplicable (nothing ran — no membership filter, or phase A had no - /// selective answer): the caller interprets the subtree as before. When - /// phase A DID run but full two-phase declines — the build side is - /// larger than the candidate set (two-phase forces build-on-left; the - /// ordinary side-picking may do better) or a key type surprise — its - /// selection is reused to materialize `right_` (RightMaterialized) - /// rather than thrown away: recomputing it from scratch was measured at - /// +12% on q03. - auto try_two_phase_probe(const DynamicScanFilter& slot, TwoPhase& outcome) - -> std::optional { - outcome = TwoPhase::NotApplicable; - if (!slot.has_membership() || !left_materialized_.has_value()) { - return std::nullopt; - } - const Table& build = *left_materialized_; - const auto* build_entry = build.find_entry(keys_->front().left); - if (build_entry == nullptr || - !std::holds_alternative>(*build_entry->column)) { - return std::nullopt; - } - - auto phase = deferred_scan_key_selection(*deferred_probe_, *deferred_exec_); - if (!phase.has_value()) { - return std::move(phase.error()); - } - if (!phase->has_value()) { - return std::nullopt; - } - auto sel = std::move(**phase); - const auto* keys_col = std::get_if>(&*sel.keys.column); - if (build.rows() > sel.selected.size() || keys_col == nullptr) { - auto right_rows = materialize_deferred_scan_rows(*deferred_probe_, sel.selected, - *deferred_exec_, std::move(sel.keys)); - if (!right_rows.has_value()) { - return std::move(right_rows.error()); - } - if (auto err = interpret_wrapped_right(std::move(*right_rows))) { - return err; - } - outcome = TwoPhase::RightMaterialized; - return std::nullopt; - } - - // Publish only: this path builds on the left and scans the right, but - // it emits one precomputed table rather than running either streaming - // shape, so it takes no operator mode from the orientation. - if (auto published = publish_build( - build_join_side(build, keys_->front().left, ExprType::Int, - JoinOrientation::BuildLeft, build_partitions(build.rows()))); - !published.has_value()) { - return std::move(published.error()); - } - - // Probe the candidate keys in scan order; record one hit per - // surviving row plus the expanded (build row, survivor) pairs — the - // same probe-order-major layout emit_swapped produces. - const auto* key_data = keys_col->data(); - const ValidityBitmap* key_validity = - sel.keys.validity.has_value() ? &*sel.keys.validity : nullptr; - const std::size_t n = keys_col->size(); - - // Same scan/replay shape as `probe_swapped`, with a twist: `ri` here - // indexes HITS (the survivor list), not probe rows, so each part - // needs two prefix offsets — its first hit index and its first output - // pair — before the replays can write disjoint slices. One part when - // the gate declines, so the serial path is the same code. - const auto scan = [&](std::size_t begin, std::size_t end, - std::vector& hits, std::size_t& total) { - for (std::size_t i = begin; i < end; ++i) { - if (key_validity != nullptr && !(*key_validity)[i]) { - continue; - } - const std::size_t head = probe_.index().i64_heads.find_head(key_data[i]); - if (head == kNil) { - continue; - } - hits.push_back(JoinProbe::SwappedHit{.rrow = i, .head = head}); - for (std::size_t cur = head; cur != kNil; cur = probe_.index().chain_next[cur]) { - ++total; - } - } - }; - const std::size_t workers = probe_.probe_parallel_workers(n); - if (workers == 0) { - probe_.swapped_parts_.resize(1); - probe_.swapped_parts_[0].hits.clear(); - probe_.swapped_parts_[0].total = 0; - scan(0, n, probe_.swapped_parts_[0].hits, probe_.swapped_parts_[0].total); - } else { - auto& pool = process_worker_pool(); - const std::size_t grain = (n + workers - 1) / workers; - probe_.swapped_parts_.resize(workers); - auto batch = pool.submit(workers, [&](std::size_t w) { - auto& part = probe_.swapped_parts_[w]; - part.hits.clear(); - part.total = 0; - const std::size_t begin = w * grain; - const std::size_t end = std::min(n, begin + grain); - if (begin < end) { - scan(begin, end, part.hits, part.total); - } - }); - batch.wait(); - } - const std::size_t n_parts = probe_.swapped_parts_.size(); - std::vector hit_offsets(n_parts); - probe_.part_offsets_.resize(n_parts); - std::size_t n_hits = 0; - std::size_t total = 0; - for (std::size_t w = 0; w < n_parts; ++w) { - hit_offsets[w] = n_hits; - probe_.part_offsets_[w] = total; - n_hits += probe_.swapped_parts_[w].hits.size(); - total += probe_.swapped_parts_[w].total; - } - - Selection survivors(n_hits); - std::vector li(total, 0); - std::vector ri(total, 0); - Column gathered_keys; - const bool gather_keys = n_hits != n; - if (gather_keys) { - gathered_keys.resize_for_overwrite(n_hits); - } - // Detach once here, not per element inside the replay: the mutable - // `operator[]` pays a CoW check every call, and on a worker the - // detach itself would race. - std::int64_t* gathered_out = gather_keys ? gathered_keys.data() : nullptr; - const auto replay = [&](std::size_t w) { - const auto& part = probe_.swapped_parts_[w]; - std::size_t h = hit_offsets[w]; - std::size_t pos = probe_.part_offsets_[w]; - for (const JoinProbe::SwappedHit& hit : part.hits) { - survivors[h] = sel.selected[hit.rrow]; - if (gathered_out != nullptr) { - gathered_out[h] = key_data[hit.rrow]; - } - for (std::size_t cur = hit.head; cur != kNil; - cur = probe_.index().chain_next[cur]) { - li[pos] = cur; - ri[pos] = h; - ++pos; - } - ++h; - } - }; - if (workers == 0) { - replay(0); - } else { - auto batch = process_worker_pool().submit(n_parts, replay); - batch.wait(); - if (deferred_exec_->parallel_stats != nullptr) { - deferred_exec_->parallel_stats->parallel_probes.fetch_add( - 1, std::memory_order_relaxed); - } - } - const bool ri_identity = total == n_hits; - - // Survivors' key values, gathered in memory from phase A's keys. - ColumnEntry key_entry; - key_entry.name = sel.keys.name; - if (!gather_keys) { - key_entry.column = sel.keys.column; - key_entry.validity = sel.keys.validity; - } else { - key_entry.column = std::make_shared(std::move(gathered_keys)); - // Null keys never match, so every survivor's key is valid. - } - - auto right_rows = materialize_deferred_scan_rows(*deferred_probe_, survivors, - *deferred_exec_, std::move(key_entry)); - if (!right_rows.has_value()) { - return std::move(right_rows.error()); - } - if (auto err = interpret_wrapped_right(std::move(*right_rows))) { - return err; - } - - Table left_copy; - left_copy.columns.reserve(build.columns.size()); - for (const auto& c : build.columns) { - left_copy.add_column(c.name, *c.column); - left_copy.columns.back().validity = c.validity; - } - auto out = probe_.assemble_output(std::move(left_copy), li.data(), ri.data(), total, - /*li_identity=*/false, ri_identity); - if (!out.has_value()) { - return std::move(out.error()); - } - precomputed_output_ = std::move(*out); - outcome = TwoPhase::Precomputed; - return std::nullopt; - } - - // Derive the probe scan's dynamic filter from the build side's valid key - // values (int keys only; other key types publish nothing). Sound for any - // inner join regardless of which side ends up as the build: a probe row - // whose key fails the filter cannot match. - // - // Everything here is published ungated — membership because a range - // estimate cannot predict set selectivity (the scan decides with a - // sampled pass rate), and min/max because the consumer owns the policy: - // materialize_deferred_scan gates conjunct synthesis on estimated - // pruning, the fused key scan uses the raw bounds for row-group - // skipping. - void publish_build_filter(const Table& build, DynamicScanFilter& slot) const { - publish_build_filter_column(build, keys_->front().left, slot); - } - - // Component-selecting variant for the two-key deferred-probe POC - // (`resolve_deferred_probe_pair`): publishes a filter over exactly one - // named build-side column instead of always `keys_->front().left`, since - // the pair join's scan filter only ever covers one of the two keys. - static void publish_build_filter_column(const Table& build, const std::string& key_name, - DynamicScanFilter& slot) { - const auto* entry = build.find_entry(key_name); - if (entry == nullptr) { - return; - } - const auto* col = std::get_if>(&*entry->column); - if (col == nullptr || col->empty()) { - return; - } - const ValidityBitmap* validity = entry->validity.has_value() ? &*entry->validity : nullptr; - const auto* data = col->data(); - const std::size_t n = col->size(); - std::int64_t mn = std::numeric_limits::max(); - std::int64_t mx = std::numeric_limits::min(); - std::size_t valid_rows = 0; - for (std::size_t r = 0; r < n; ++r) { - if (validity != nullptr && !(*validity)[r]) { - continue; - } - mn = std::min(mn, data[r]); - mx = std::max(mx, data[r]); - ++valid_rows; - } - if (valid_rows == 0) { - return; - } - - // Every build side gets a Bloom — even alongside an exact list, the - // Bloom is the probe fast path (see DynamicScanFilter::passes). - // Duplicate inserts are harmless. A small build side (dimension - // chains: nation, region, filtered part) additionally dedups cheaply - // into an exact list, cancelling the Bloom's false positives. - constexpr std::size_t kInListBuildMax = 4096; - constexpr std::size_t kInListMax = 1024; - JoinBloomFilter bloom(valid_rows); - for (std::size_t r = 0; r < n; ++r) { - if (validity != nullptr && !(*validity)[r]) { - continue; - } - bloom.insert(data[r]); - } - slot.bloom = std::move(bloom); - if (valid_rows <= kInListBuildMax) { - std::vector keys; - keys.reserve(valid_rows); - for (std::size_t r = 0; r < n; ++r) { - if (validity != nullptr && !(*validity)[r]) { - continue; - } - keys.push_back(data[r]); - } - std::ranges::sort(keys); - keys.erase(std::ranges::unique(keys).begin(), keys.end()); - if (keys.size() <= kInListMax) { - slot.in_list = std::move(keys); - } - } - // Raw facts, not policy: whether these bounds are worth acting on is - // the consumer's call — materialize_deferred_scan gates synthesized - // conjuncts on estimated pruning, while the fused key scan uses them - // ungated to skip whole row groups (which has no gather downside). - slot.min = mn; - slot.max = mx; - } - - /// Would indexing the right instead of the left buy the pending `order`, - /// and is the index small enough that it is worth buying? - /// - /// The pending keys are in the join's output names and the left's claim is - /// in the left's own, so the claim is restated through the output plan - /// before they are compared -- a suffixed key is renamed and a key the - /// output drops takes the claim with it. - auto order_preserving_pays(const Table& left_table, std::size_t n_left, std::size_t n_right) - -> bool { - constexpr std::size_t kMaxOrderPreservingBuildRatio = 4; - const auto& left_ordering = left_table.properties().ordering(); - if (pending_order_ == nullptr || pending_order_->empty() || !left_ordering.has_value() || - n_right > kMaxOrderPreservingBuildRatio * n_left) { - return false; - } - if (!probe_.right_emit_ready_) { - if (auto ready = probe_.setup_right_emit_schema(left_table); !ready.has_value()) { - return false; // the join is about to fail on this anyway - } - } - std::vector carried; - for (const auto& key : *left_ordering) { - std::size_t idx = left_table.columns.size(); - for (std::size_t i = 0; i < left_table.columns.size(); ++i) { - if (left_table.columns[i].name == key.name) { - idx = i; - break; - } - } - if (idx == left_table.columns.size() || idx >= probe_.left_emit_names_.size()) { - return false; - } - carried.push_back( - ir::OrderKey{.name = probe_.left_emit_names_[idx], .ascending = key.ascending}); - } - return TableProperties::sorted_by(std::move(carried)).satisfies(*pending_order_); - } - - OperatorPtr left_; - /// The right side, owned jointly with every probe reading it. A - /// `shared_ptr` rather than a member `Table` because a probe must be able - /// to outlive this operator and several probes must be able to read one - /// build side at once -- see `JoinProbe::right_`. Written only by the - /// build phase, which finishes before any probe runs. - std::shared_ptr
right_ = std::make_shared
(); - const std::vector* keys_; - /// The probe half. The operator runs the build and decides which side to - /// index; everything after that belongs here. Moved into `probe_op_` when - /// the orientation is BuildRight, since from then on the probe is an - /// operator of its own and this class is build-side only. - JoinProbe probe_; - /// The probe side, drained by the build phase and held until either - /// `ensure_probe_op` turns it into a source or `take_fusible_probe` hands - /// it to a pipeline above. Empty when the probe side streams from `left_`. - std::optional
probe_side_; - /// Stream mode's probe, constructed on first use: one - /// `JoinProbeOperator` over the probe-side child, or -- when that child - /// was already materialized and is big enough -- a morsel pipeline of - /// several of them over its morsels. Null in the Swapped and Precomputed - /// modes, which emit one table rather than streaming. - OperatorPtr probe_op_; - - // Deferred-probe context (see the second constructor). `deferred_probe_` - // doubles as the mode flag: non-null until the probe scan is resolved. - const DeferredScan* deferred_probe_ = nullptr; - std::string deferred_probe_name_; - const ir::Node* deferred_right_node_ = nullptr; - const TableRegistry* deferred_registry_ = nullptr; - const ScalarRegistry* deferred_scalars_ = nullptr; - const ExternRegistry* deferred_externs_ = nullptr; - const ExecutionContext* deferred_exec_ = nullptr; - bool initialized_ = false; - Mode mode_ = Mode::Stream; - - /// Both fan-out phases' parallelism, resolved by the caller - /// (`resolved_join_parallelism`, shared by every construction site). - /// `build_partitions` reads `par_.build`; `bind_probe` copies `par_.probe` - /// into the probe. src/runtime/PARALLELISM.md. Default-constructed - /// (`worker_cap == 0`) only on a hand-built operator, where both phases then - /// stay serial. - physical::JoinParallelism par_{}; - - // What an `order` above this join will ask for, or null. Only ever shifts - // which side is indexed; see `initialize`. - const std::vector* pending_order_ = nullptr; - - // Stream mode: when right > threshold and left >= right, left was - // materialized to measure but not swapped; replay as a single chunk. - std::optional
left_materialized_; - bool use_materialized_left_ = false; - std::optional
empty_schema_; - - // Swapped mode: materialized left held for later gather. - std::optional
left_table_; - bool swapped_emitted_ = false; - - // Precomputed mode: the two-phase deferred probe assembled the whole - // join output during initialization. - Table precomputed_output_; -}; - -/// Streaming hash aggregate. Maintains a `robin_hood` group index and -/// per-group `AggState` across chunks: each incoming chunk updates the -/// state per row, the chunk is released, and the final result is -/// emitted as a single output chunk on EOF. -/// -/// Eligibility is gated at `build_operator` time to the common subset -/// that streams cleanly: `Count`, `Sum`, `Min`, `Max`, `Mean` on -/// numeric (int/double) inputs. Nullable agg inputs are handled — null -/// rows skip the update, and an all-null group emits a null result. -/// Nullable group-by columns are not supported yet; they fall back to -/// `aggregate_table` via `interpret_node`. Complex aggs (Median, etc.) -/// and string aggs also fall back. -/// -/// The first chunk's group-by column types are snapshotted (including -/// the Categorical dictionary pointer when applicable) and reused when -/// building output; the chunked csv source shares dictionaries across -/// chunks, matching MaterializeOperator's existing assumption. -/// A growable array of trivially-copyable slots that grows through `realloc`. -/// -/// `std::vector` cannot use `realloc`: it must allocate, copy, and free, and on -/// this array that copy IS the cost. A group-by discovers its groups a chunk at -/// a time, so the slot array is resized once per chunk and never shrinks; by -/// the last chunk the copies dominate. Measured on q18 (3M groups over 6 -/// chunks): `size_group_arrays` cost 79ms, and pre-reserving the final size -- -/// which a real query cannot do, since the group count is what it is about to -/// find out -- removed 49ms of it. That removed cost is all copying. -/// -/// At these sizes the block is served by `mmap`, and `realloc` extends it with -/// `mremap`: page-table work, no bytes moved. The elements it must still touch -/// are only the NEW ones, which is the irreducible part. -/// -/// Deliberately minimal: no shrink, no insert, no iterators. It is a slot array -/// indexed by group id, and every use it has is `resize` / `data` / `[]`. -/// -/// The `realloc`/`free` calls are the whole reason this class exists (in-place -/// `mremap` growth, no copy) -- it is itself the RAII wrapper the check wants. -// NOLINTBEGIN(cppcoreguidelines-no-malloc) -template -class SlotArray { - public: - static_assert(std::is_trivially_copyable_v); - static_assert(std::is_trivially_destructible_v); - - SlotArray() = default; - SlotArray(const SlotArray&) = delete; - auto operator=(const SlotArray&) -> SlotArray& = delete; - SlotArray(SlotArray&& other) noexcept - : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { - other.data_ = nullptr; - other.size_ = 0; - other.capacity_ = 0; - } - auto operator=(SlotArray&& other) noexcept -> SlotArray& { - if (this != &other) { - std::free(data_); - data_ = other.data_; - size_ = other.size_; - capacity_ = other.capacity_; - other.data_ = nullptr; - other.size_ = 0; - other.capacity_ = 0; - } - return *this; - } - ~SlotArray() { std::free(data_); } - - [[nodiscard]] auto size() const noexcept -> std::size_t { return size_; } - [[nodiscard]] auto data() noexcept -> T* { return data_; } - [[nodiscard]] auto data() const noexcept -> const T* { return data_; } - auto operator[](std::size_t i) noexcept -> T& { return data_[i]; } - auto operator[](std::size_t i) const noexcept -> const T& { return data_[i]; } - - /// Grow to `n` WITHOUT initializing the new tail, which is returned for the - /// caller to fill. Never shrinks the allocation: a group-by only ever adds - /// groups. - /// - /// Split out of `resize` because on a large array the fill is the expensive - /// half and it is not serial by nature. `mremap` hands back pages the kernel - /// has yet to materialize, so writing them is 72MB of first-touch page - /// faults on q18's 3M slots — 44ms in a cold process, several times what the - /// bytes alone cost, and page faults are what scales with threads. A caller - /// holding a worker pool fans this out; one without it calls `resize` and - /// pays the serial fill. - /// - /// How much this is worth depends on whether the pages are fresh, so read - /// the two numbers separately. Cold (`ibex query.ibex`, the path a script - /// takes) the fan-out takes q18's fill 44ms -> 34ms and the whole query - /// -4.3%. Warm — the PDS-H harness, which reuses one process, so the - /// allocator hands back pages already faulted — the fill is plain bandwidth - /// and the suite geomean does not move. - [[nodiscard]] auto grow_uninitialized(std::size_t n) -> std::span { - if (n <= size_) { - size_ = n; - return {}; - } - if (n > capacity_) { - // Geometric, so a per-chunk resize does not call realloc once per - // chunk on a stream of many small chunks. - const std::size_t want = std::max(n, capacity_ + (capacity_ / 2)); - auto* grown = static_cast(std::realloc(data_, want * sizeof(T))); - if (grown == nullptr) { - throw std::bad_alloc(); - } - data_ = grown; - capacity_ = want; - } - const std::size_t old = size_; - size_ = n; - return {data_ + old, n - old}; - } - - /// Value-initialize `tail`, a range `grow_uninitialized` just handed back. - /// - /// One `memset` when `T`'s value-initialized form is all-zero bytes, which - /// every slot type here is (`AggSlotCore`'s two enums both start at 0). The - /// per-element copy this replaces cost q18's fill 12ms of its 55: three - /// million 24-byte `memcpy`s the compiler will not fuse, because it cannot - /// see that the prototype is zeros. - /// - /// Padding is why the test is a run-time `memcmp` rather than a - /// `static_assert`: value-initialization zeroes `T`'s padding too, so the - /// comparison is well defined here, but no constant expression can state - /// that for a type with padding. The loop keeps the class honest for a - /// future slot type whose default is not all zeros. - static void fill_default(std::span tail) noexcept { - if (tail.empty()) { - return; - } - const T prototype{}; - alignas(T) std::array zero{}; - // prototype{} zeroes padding too (see the class comment), so this is well defined. - // NOLINTNEXTLINE(cert-exp42-c,cert-flp37-c,bugprone-suspicious-memory-comparison) - if (std::memcmp(&prototype, zero.data(), sizeof(T)) == 0) { - // Through `void*`: `T` has default member initializers, so it is not - // trivially default-constructible and -Wclass-memaccess objects to - // memset-ing it directly. The memcmp above is what licenses this. - std::memset(static_cast(tail.data()), 0, tail.size() * sizeof(T)); - return; - } - for (auto& slot : tail) { - std::memcpy(&slot, &prototype, sizeof(T)); - } - } - - /// Grow to `n`, value-initializing the new tail on the calling thread. - void resize(std::size_t n) { fill_default(grow_uninitialized(n)); } - - private: - T* data_ = nullptr; - std::size_t size_ = 0; - std::size_t capacity_ = 0; -}; -// NOLINTEND(cppcoreguidelines-no-malloc) - -class ChunkedAggregateOperator final : public Operator { - public: - /// `Cat` carries a Categorical's *code*, which the pair path may treat as - /// an integer for the same reason `process_rows_cat` may index an array - /// with it: within one operator a dictionary only ever grows and never - /// reorders, so a code identifies the same value in every chunk. - enum class IntKeyKind : std::uint8_t { Int64, Date, Ts, Cat }; - - ChunkedAggregateOperator(OperatorPtr child, const std::vector* group_by, - const std::vector* aggregations, - const ExecutionContext& exec, physical::AggregateParallelism par = {}) - : child_(std::move(child)), - group_by_(group_by), - aggregations_(aggregations), - exec_(&exec), - par_(par) {} - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (emitted_) { - return std::optional{}; - } - while (true) { - auto chunk_res = child_->next(); - if (!chunk_res.has_value()) { - return std::unexpected(std::move(chunk_res.error())); - } - if (!chunk_res.value().has_value()) { - break; - } - const Chunk chunk = std::move(*chunk_res.value()); - if (auto err = process_chunk(chunk)) { - return std::unexpected(*err); - } - // `chunk` goes out of scope here, releasing its memory - // before we pull the next one from the child. - } - emitted_ = true; - return build_output_chunk(); - } - - private: - /// Telemetry for the breaker-parallelism slice (src/runtime/PARALLELISM.md). - /// The plan owns each phase's worker cap and fan-out permission now; the - /// fan-out output is byte-identical to serial, so without a counter a gate - /// that silently stopped matching would lose the parallelism with every - /// test green. Counted once per fan-out commit (per chunk for `partition`, - /// once for `finalize`, which runs once). - void note_partition_fanout() const { - if (exec_ != nullptr && exec_->parallel_stats != nullptr) { - exec_->parallel_stats->parallel_aggregate_partitions.fetch_add( - 1, std::memory_order_relaxed); - } - } - void note_finalize_fanout() const { - if (exec_ != nullptr && exec_->parallel_stats != nullptr) { - exec_->parallel_stats->parallel_aggregate_finalizes.fetch_add( - 1, std::memory_order_relaxed); - } - } - - auto process_chunk(const Chunk& chunk) -> std::optional { - if (std::getenv("IBEX_AGG_PARTITION_DEBUG") != nullptr) { - ibex::formatting::print(stderr, "[agg_process_chunk] rows={} group_by_size={}\n", - chunk.rows(), group_by_->size()); - } - // Counted here, once per chunk, because the partition gate below asks - // how much input this OPERATOR has — a question the per-call row count - // stopped answering the moment sources began arriving in pieces. - rows_offered_ += chunk.rows(); - std::vector group_entries; - group_entries.reserve(group_by_->size()); - for (const auto& key : *group_by_) { - const ColumnEntry* entry = nullptr; - for (const auto& e : chunk.columns) { - if (e.name == key.name) { - entry = &e; - break; - } - } - if (entry == nullptr) { - return "group-by column not found: " + key.name; - } - group_entries.push_back(entry); - } - - std::vector agg_entries(aggregations_->size(), nullptr); - for (std::size_t i = 0; i < aggregations_->size(); ++i) { - const auto& agg = (*aggregations_)[i]; - if (agg.func == ir::AggFunc::Count) { - continue; - } - const ColumnEntry* entry = nullptr; - for (const auto& e : chunk.columns) { - if (e.name == agg.column.name) { - entry = &e; - break; - } - } - if (entry == nullptr) { - return "aggregate column not found: " + agg.column.name; - } - const ExprType kind = expr_type_for_column(*entry->column); - const bool first_or_last = - agg.func == ir::AggFunc::First || agg.func == ir::AggFunc::Last; - // First/Last also accept String (which covers Column and - // Column — expr_type_for_column collapses both to - // String); every other function stays numeric-only. - const bool supported = kind == ExprType::Int || kind == ExprType::Double || - (first_or_last && kind == ExprType::String); - if (!supported) { - return "ChunkedAggregateOperator: non-numeric aggregation not supported"; - } - agg_entries[i] = entry; - } - - if (!initialized_) { - n_aggs_ = aggregations_->size(); - plan_.reserve(n_aggs_); - for (std::size_t i = 0; i < n_aggs_; ++i) { - SlotPlan p; - p.func = (*aggregations_)[i].func; - if (p.func == ir::AggFunc::Count) { - p.kind = ExprType::Int; - } else { - p.kind = expr_type_for_column(*agg_entries[i]->column); - p.categorical = - std::holds_alternative>(*agg_entries[i]->column); - } - plan_.push_back(p); - } - // Lay the scratch out once the plan is known. Skew/Kurtosis share - // one online recurrence that updates both higher moments, so each - // asks for the pair. - scratch_offset_.assign(n_aggs_, 0); - scratch_stride_ = 0; - for (std::size_t i = 0; i < n_aggs_; ++i) { - // Scratch layout is [m2, m3, m4]. Stddev needs only the - // first; the higher moments imply it, since their recurrence - // reads m2 on every update. - if (plan_[i].func == ir::AggFunc::Stddev) { - plan_[i].scratch_doubles = 1; - } else if (plan_[i].func == ir::AggFunc::Skew || - plan_[i].func == ir::AggFunc::Kurtosis) { - plan_[i].scratch_doubles = 3; - } - scratch_offset_[i] = static_cast(scratch_stride_); - scratch_stride_ += plan_[i].scratch_doubles; - } - group_templates_.reserve(group_entries.size()); - bool all_cat = true; - for (const auto* e : group_entries) { - group_templates_.push_back(make_empty_like(*e->column)); - if (!std::holds_alternative>(*e->column) || - e->validity.has_value()) { - all_cat = false; - } - } - cat_fast_path_ = all_cat && !group_entries.empty(); - // Single-string-key fast path: avoids the generic `Key`/ScalarValue - // variant path used by `process_rows_generic`. High-cardinality - // `sum by user_id` (~100K distinct strings in 2M rows) was spending - // most of its time constructing per-row ScalarValue variants and - // hashing them; the string path uses a string_view map keyed against - // an owned char/offset dictionary instead. - str_fast_path_ = - group_entries.size() == 1 && - std::holds_alternative>(*group_entries[0]->column) && - !group_entries[0]->validity.has_value(); - // Single fixed-width-integer key: a direct value map, no owned Key. - const auto int_kind_of = [](const ColumnValue& col) -> std::optional { - if (std::holds_alternative>(col)) { - return IntKeyKind::Int64; - } - if (std::holds_alternative>(col)) { - return IntKeyKind::Date; - } - if (std::holds_alternative>(col)) { - return IntKeyKind::Ts; - } - return std::nullopt; - }; - if (group_entries.size() == 1 && !group_entries[0]->validity.has_value()) { - if (auto kind = int_kind_of(*group_entries[0]->column)) { - int_fast_path_ = true; - int_key_kind_ = *kind; - } - } else if (!cat_fast_path_ && group_entries.size() == 2 && - !group_entries[0]->validity.has_value() && - !group_entries[1]->validity.has_value()) { - // A Categorical joins the pair path as its code. `cat_fast_path_` - // already owns the all-Categorical case and is dispatched first, - // so this is reached only by a *mixed* pair — `by { symbol, day }` - // over a Categorical and a Date, which otherwise fell to the - // generic path and hashed the symbol as text once per row. - const auto pair_kind_of = [&](const ColumnValue& col) -> std::optional { - if (std::holds_alternative>(col)) { - return IntKeyKind::Cat; - } - return int_kind_of(col); - }; - auto ka = pair_kind_of(*group_entries[0]->column); - auto kb = pair_kind_of(*group_entries[1]->column); - if (ka.has_value() && kb.has_value()) { - pair_int_fast_path_ = true; - int_key_kind_ = *ka; - int_key_kind_b_ = *kb; - const auto is_32_bit = [](IntKeyKind k) { - return k == IntKeyKind::Cat || k == IntKeyKind::Date; - }; - pair_packs_u64_ = is_32_bit(*ka) && is_32_bit(*kb); - } - } else if (group_entries.size() >= 3) { - // Three or more keys had no fast path at all: the branches above - // only recognise one key or two, so everything wider fell to - // `process_rows_generic`, which hashes a KeyCol tuple per row - // and hashes a Categorical as TEXT while doing it. If the whole - // key packs into a flat integer, `process_rows_packed` replaces - // that with one hash of a POD — and, because a packed key is - // something `try_discover_partitioned` can carry, threads the - // discovery too. - // - // The probe is discarded; the real plan is rebuilt per chunk, - // since a Categorical's remap is only valid for its own chunk. - packed_fast_path_ = encoder_.build_packed_key(group_entries).has_value(); - } - initialized_ = true; - } else { - for (std::size_t i = 0; i < n_aggs_; ++i) { - if (plan_[i].func == ir::AggFunc::Count) { - continue; - } - const ExprType kind = expr_type_for_column(*agg_entries[i]->column); - if (kind != plan_[i].kind) { - return "ChunkedAggregateOperator: aggregate column type changed across chunks"; - } - } - for (std::size_t i = 0; i < group_entries.size(); ++i) { - if (group_entries[i]->column->index() != group_templates_[i].index()) { - return "ChunkedAggregateOperator: group-by column type changed across chunks"; - } - } - } - - const std::size_t rows = chunk.rows(); - - // Global aggregate (`select { … }` with no `by`). Every row belongs to - // group 0, so the generic path below was running a hash probe per row - // against an EMPTY key just to rediscover that. Accumulate straight - // into the single group, and — since the groups are independent of row - // order — fan the row range out across workers. - if (group_entries.empty()) { - return process_rows_ungrouped(agg_entries, rows); - } - // A fast-path index records only raw values/codes. It therefore cannot - // distinguish a later null from that value's zero/code representation. - // Parquet commonly omits an all-valid row group's bitmap, so this is a - // real streaming transition rather than a schema change visible in the - // first chunk. - // - // Every fast path stores its groups' raw values in a form the generic - // `KeyRowIndex` can be reseeded from -- `int_order_`/`str_order_` - // directly, `cat_order_`/`multi_cat_codes_flat_` via the dictionary - // `group_templates_` still holds, `pair_order_` via both, and the - // packed path's `group_order_` is already boxed `Key`s (see - // `migrate_packed_fast_path_to_generic`). Migrating only rebuilds the - // key->gid lookup; the accumulated `flat_slots_`/`scratch_` those gids - // already own are untouched, and this chunk then runs the generic path - // below like any other. - if ((cat_fast_path_ || str_fast_path_ || int_fast_path_ || pair_int_fast_path_ || - packed_fast_path_) && - std::ranges::any_of(group_entries, [](const ColumnEntry* entry) { - return entry->validity.has_value(); - })) { - if (cat_fast_path_) { - migrate_cat_fast_path_to_generic(group_entries.size()); - } else if (int_fast_path_) { - migrate_int_fast_path_to_generic(); - } else if (str_fast_path_) { - migrate_str_fast_path_to_generic(); - } else if (pair_int_fast_path_) { - migrate_pair_int_fast_path_to_generic(); - } else if (packed_fast_path_) { - migrate_packed_fast_path_to_generic(); - } - } - if (cat_fast_path_) { - return process_rows_cat(group_entries, agg_entries, rows); - } - if (str_fast_path_) { - return process_rows_str(group_entries, agg_entries, rows); - } - if (int_fast_path_) { - return process_rows_int(group_entries, agg_entries, rows); - } - if (pair_int_fast_path_) { - return process_rows_int_pair(group_entries, agg_entries, rows); - } - if (packed_fast_path_ && rows != 0) { - auto plan = encoder_.build_packed_key(group_entries); - if (!plan.has_value()) { - // The shape was packable when the first chunk fixed the path, - // so this is an unsupported mid-stream key-layout transition. - return "ChunkedAggregateOperator: group-by key column gained nulls across chunks"; - } - if (plan->width <= sizeof(std::uint64_t)) { - return process_rows_packed(group_entries, agg_entries, plan->cols, rows, packed64_); - } - if (plan->width <= sizeof(PackedKeyEncoder::Packed128)) { - return process_rows_packed(group_entries, agg_entries, plan->cols, rows, - packed128_); - } - return process_rows_packed(group_entries, agg_entries, plan->cols, rows, packed256_); - } - return process_rows_generic(group_entries, agg_entries, rows); - } - - auto process_rows_str(const std::vector& group_entries, - const std::vector& agg_entries, std::size_t rows) - -> std::optional { - const auto& col = std::get>(*group_entries[0]->column); - const char* src_chars = col.chars_data(); - const std::uint32_t* src_off = col.offsets_data(); - - gids_buf_.resize(rows); - auto* gids = gids_buf_.data(); - - // High-cardinality string keys are where this path pays: discovery is - // the serial half, and a string group-by has nothing else to hide it - // behind. Probing with a view keeps the owning copy per GROUP, as the - // serial loop below does. - const auto key_at = [&](std::size_t row) -> std::string_view { - return std::string_view{src_chars + src_off[row], src_off[row + 1] - src_off[row]}; - }; - if (try_discover_partitioned( - key_at, rows, gids, str_partitions_, [&](std::size_t n) { str_order_.resize(n); }, - [&](const std::string& key, std::uint32_t gid, std::size_t) { - str_order_[gid] = key; - }, - kDefaultPartitionMinRows, - [&](std::uint32_t gid) -> std::string_view { return str_order_[gid]; })) { - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - // Run-length shortcut: sorted or chunked CSV often has adjacent - // repeats; skip the hash lookup when the key matches the previous row. - std::string_view prev_key; - std::uint32_t prev_gid = std::numeric_limits::max(); - for (std::size_t row = 0; row < rows; ++row) { - const std::string_view key{src_chars + src_off[row], src_off[row + 1] - src_off[row]}; - std::uint32_t gid{}; - if (key == prev_key) { - gid = prev_gid; - } else { - // Transparent lookup on string_view avoids constructing a - // std::string per probe. Insertions pay one std::string - // construction per novel key — with libstdc++'s 15-char SSO, - // 11-char user_id strings stay inline (no heap alloc). - auto it = str_index_.find(key); - if (it == str_index_.end()) { - gid = static_cast(n_groups_); - str_index_.emplace(std::string(key), gid); - str_order_.emplace_back(key); - ++n_groups_; - size_group_arrays(); - } else { - gid = static_cast(it->second); - } - prev_key = key; - prev_gid = gid; - } - gids[row] = gid; - } - - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - // Single fixed-width-integer key: probe a value -> gid map directly, the way - // process_rows_str does for strings. Date/Timestamp are read as their raw - // integer (days / nanos), which is order- and equality-faithful. - auto process_rows_int(const std::vector& group_entries, - const std::vector& agg_entries, std::size_t rows) - -> std::optional { - const ColumnValue& key_col = *group_entries[0]->column; - const std::int64_t* i64 = nullptr; - const Date* dates = nullptr; - const Timestamp* stamps = nullptr; - switch (int_key_kind_) { - case IntKeyKind::Int64: - i64 = std::get>(key_col).data(); - break; - case IntKeyKind::Date: - dates = std::get>(key_col).data(); - break; - case IntKeyKind::Ts: - stamps = std::get>(key_col).data(); - break; - case IntKeyKind::Cat: - // A lone Categorical key never selects this path: it is - // all-Categorical by definition, so `cat_fast_path_` claims it - // and dispatches first. Only the pair path admits `Cat`. - return "ChunkedAggregateOperator: categorical key on the single-int path"; - } - const auto key_at = [&](std::size_t row) -> std::int64_t { - switch (int_key_kind_) { - case IntKeyKind::Int64: - return i64[row]; - case IntKeyKind::Date: - return dates[row].days; - case IntKeyKind::Ts: - return stamps[row].nanos; - case IntKeyKind::Cat: - break; - } - return 0; - }; - - // The q18 shape (one Int64 key and one Double sum) is a streaming sink, - // not a sequence of per-chunk fork/join pipelines. Each chunk becomes - // one independent hot-table task; the caller immediately pulls the - // next chunk, and all tasks join once at end-of-stream. Besides removing - // the three barriers per chunk, this keeps the common clustered key in - // a 4096-slot cache-resident reduction and sends only cold/pre-aggregated - // records to the persistent partition maps. - if (try_async_hot_int_sum(group_entries[0]->column, agg_entries, rows)) { - return std::nullopt; - } - - gids_buf_.resize(rows); - auto* gids = gids_buf_.data(); - - // A non-null First value is fixed at the same row that creates its - // group. Record it during discovery and omit the later all-row scan. - // Group-key reduction turns q10's six descriptive keys into exactly - // this shape; scanning 229k rows once per carried field was redundant. - if (discovery_first_eligible_.empty()) { - discovery_first_eligible_.resize(n_aggs_, 0U); - for (std::size_t a = 0; a < n_aggs_; ++a) { - discovery_first_eligible_[a] = plan_[a].func == ir::AggFunc::First ? 1U : 0U; - } - } - std::vector discovery_first(n_aggs_, 0U); - bool has_discovery_first = false; - if (std::getenv("IBEX_DISABLE_DISCOVERY_FIRST") == nullptr) { - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (discovery_first_eligible_[a] == 0U) { - continue; - } - if (agg_entries[a]->validity.has_value()) { - // A group may still be waiting for its first non-null - // value. Keep this field on the ordinary scan for every - // later chunk too, even if that later chunk has no nulls. - discovery_first_eligible_[a] = 0U; - continue; - } - discovery_first[a] = 1U; - has_discovery_first = true; - } - } - const std::size_t groups_before = n_groups_; - std::vector first_rows; - - // Partition-owned accumulation, the same shape the PairIntKey path - // above takes. It fuses discovery and the sum/count into one pass over - // partition-local state, so the global first-occurrence numbering -- - // and with it the whole second `accumulate_gids` scan of every row -- - // is deferred to a single merge over GROUPS at emission. - // - // `int_order_` holds the raw key whatever `int_key_kind_` is; the emit - // side reconstructs Date/Timestamp/Categorical from it exactly as it - // does for the ordinary int path, so this needs no kind-specific arm. - if (try_owned>( - key_at, rows, gids, agg_entries, owned_int_partitions_, kIntOwnedMinRows)) { - return std::nullopt; - } - - if (try_discover_partitioned>( - key_at, rows, gids, int_partitions_, - [&](std::size_t n) { - int_order_.resize(n); - if (has_discovery_first) { - first_rows.resize(n - groups_before); - } - }, - [&](std::int64_t key, std::uint32_t gid, std::size_t row) { - int_order_[gid] = key; - if (has_discovery_first) { - first_rows[gid - groups_before] = row; - } - }, - kDefaultPartitionMinRows, [&](std::uint32_t gid) { return int_order_[gid]; })) { - if (has_discovery_first) { - seed_discovery_first(groups_before, first_rows, agg_entries, discovery_first); - } - accumulate_gids(gids, agg_entries, rows, - has_discovery_first ? &discovery_first : nullptr); - return std::nullopt; - } - - // Run-length shortcut, as in the string path: sorted/chunked input often - // repeats the key, so skip the map lookup when it matches the last row. - std::int64_t prev_key = 0; - std::uint32_t prev_gid = std::numeric_limits::max(); - bool have_prev = false; - for (std::size_t row = 0; row < rows; ++row) { - const std::int64_t key = key_at(row); - std::uint32_t gid{}; - if (have_prev && key == prev_key) { - gid = prev_gid; - } else { - auto it = int_index_.find(key); - if (it == int_index_.end()) { - gid = static_cast(n_groups_); - int_index_.emplace(key, gid); - int_order_.push_back(key); - ++n_groups_; - size_group_arrays(); - if (has_discovery_first) { - first_rows.push_back(row); - } - } else { - gid = it->second; - } - prev_key = key; - prev_gid = gid; - have_prev = true; - } - gids[row] = gid; - } - - if (has_discovery_first) { - seed_discovery_first(groups_before, first_rows, agg_entries, discovery_first); - } - accumulate_gids(gids, agg_entries, rows, has_discovery_first ? &discovery_first : nullptr); - return std::nullopt; - } - - static constexpr std::size_t kOwnedHotSlots = 4096; - - struct OwnedHotRecord { - std::int64_t key = 0; - std::uint64_t first_row = 0; - AggSlotCore slot; - }; - - struct OwnedHotChunk { - std::shared_ptr key_column; - std::shared_ptr sum_column; - std::optional sum_validity; - std::uint64_t row_base = 0; - std::size_t rows = 0; - std::size_t part_count = 0; - std::vector> records_by_partition; - std::optional error; - }; - - struct OwnedHotSlot { - std::uint32_t tag = std::numeric_limits::max(); - std::uint32_t last_access_tag = std::numeric_limits::max(); - std::uint32_t record = std::numeric_limits::max(); - }; - - [[nodiscard]] static auto owned_hot_hash(std::int64_t key) noexcept -> std::uint64_t { - // robin_hood's Int64 hash preserves too much of a sequential key's bit - // pattern for a high-bit fixed table. SplitMix's finalizer gives both - // candidate slots and the tag independent-looking bits at tiny cost. - auto x = static_cast(key); - x ^= x >> 30U; - x *= 0xbf58476d1ce4e5b9ULL; - x ^= x >> 27U; - x *= 0x94d049bb133111ebULL; - x ^= x >> 31U; - return x; - } - - static void process_owned_hot_chunk(OwnedHotChunk& job) noexcept { - try { - const auto* keys = std::get>(*job.key_column).data(); - const auto* values = std::get>(*job.sum_column).data(); - const ValidityBitmap* validity = - job.sum_validity.has_value() ? &*job.sum_validity : nullptr; - constexpr std::uint32_t kEmpty = std::numeric_limits::max(); - constexpr unsigned kShift = 64U - 12U; - constexpr std::uint64_t kH2Mult = 0xf1357aea2e62a9c5ULL; - - std::array table{}; - std::vector records; - records.reserve(std::max(kOwnedHotSlots, job.rows / 3)); - std::size_t filled = 0; - std::uint64_t prng = 0; - std::int64_t last_key = 0; - std::uint32_t last_record = kEmpty; - bool have_last = false; - - const auto append_record = [&](std::size_t row) -> std::uint32_t { - const auto index = static_cast(records.size()); - OwnedHotRecord record; - record.key = keys[row]; - record.first_row = job.row_base + row; - if (validity == nullptr || (*validity)[row]) { - record.slot.double_value = values[row]; - record.slot.mark_present(); - } - records.push_back(record); - return index; - }; - const auto update_record = [&](std::uint32_t record, std::size_t row) { - if (validity == nullptr || (*validity)[row]) { - auto& slot = records[record].slot; - slot.double_value += values[row]; - slot.mark_present(); - } - }; - - for (std::size_t row = 0; row < job.rows; ++row) { - const std::int64_t key = keys[row]; - // HashKeys in Polars computes hashes as a column kernel before - // probing. Here keys arrive as raw Int64s, and q18's useful - // locality is specifically runs of the same key. Retain the - // most recent hot/cold record so the other rows in a run avoid - // both the SplitMix hash and the two fixed-table probes. - if (have_last && key == last_key) { - update_record(last_record, row); - continue; - } - const std::uint64_t hash = owned_hot_hash(key); - const auto tag = static_cast(hash); - const auto h1 = static_cast(hash >> kShift); - const auto h2 = static_cast((hash * kH2Mult) >> kShift); - auto& s1 = table[h1]; - auto& s2 = table[h2]; - - if (s1.tag == tag && s1.record != kEmpty && records[s1.record].key == key) { - s1.last_access_tag = tag; - update_record(s1.record, row); - last_key = key; - last_record = s1.record; - have_last = true; - continue; - } - if (s2.tag == tag && s2.record != kEmpty && records[s2.record].key == key) { - s2.last_access_tag = tag; - update_record(s2.record, row); - last_key = key; - last_record = s2.record; - have_last = true; - continue; - } - - if (filled < kOwnedHotSlots) { - OwnedHotSlot* empty = s1.record == kEmpty ? &s1 : nullptr; - if (empty == nullptr && s2.record == kEmpty) { - empty = &s2; - } - if (empty != nullptr) { - empty->tag = tag; - empty->last_access_tag = tag; - empty->record = append_record(row); - last_key = key; - last_record = empty->record; - have_last = true; - ++filled; - continue; - } - } - - // Polars' second chance: a miss first marks one candidate as - // recently considered and stays cold. Seeing the same tag - // again earns admission, evicting that candidate's mapping; - // its record already contains the complete pre-aggregate. - OwnedHotSlot& chosen = (prng >> 63U) != 0 ? s1 : s2; - prng += hash; - if (chosen.last_access_tag == tag) { - chosen.tag = tag; - chosen.last_access_tag = tag; - chosen.record = append_record(row); - last_record = chosen.record; - } else { - chosen.last_access_tag = tag; - last_record = append_record(row); - } - last_key = key; - have_last = true; - } - - // Records were appended at their first source row and only updated - // in place, so this stable routing preserves global first-seen - // order without a histogram/scatter phase or a sort. - job.records_by_partition.resize(job.part_count); - std::vector counts(job.part_count, 0); - const robin_hood::hash partition_hash; - const std::size_t mask = job.part_count - 1; - for (const auto& record : records) { - ++counts[partition_hash(record.key) & mask]; - } - for (std::size_t p = 0; p < job.part_count; ++p) { - job.records_by_partition[p].reserve(counts[p]); - } - for (auto& record : records) { - const std::size_t p = partition_hash(record.key) & mask; - job.records_by_partition[p].push_back(record); - } - - // Release decoded buffers as soon as this task is done. The job's - // compact pre-aggregates remain until the one final cold merge. - job.key_column.reset(); - job.sum_column.reset(); - job.sum_validity.reset(); - } catch (const std::exception& error) { - job.error = "async hot aggregate: " + std::string(error.what()); - } catch (...) { - job.error = "async hot aggregate: non-standard worker exception"; - } - } - - auto try_async_hot_int_sum(const std::shared_ptr& key_column, - const std::vector& agg_entries, std::size_t rows) - -> bool { - if (!owned_async_hot_mode_) { - if (std::getenv("IBEX_DISABLE_OWNED_PAIR_AGG") != nullptr || - std::getenv("IBEX_DISABLE_ASYNC_HOT_AGG") != nullptr || n_groups_ > 0 || - partitioned_active_ || owned_mode_ || n_aggs_ != 1 || - plan_[0].func != ir::AggFunc::Sum || plan_[0].kind != ExprType::Double || - int_key_kind_ != IntKeyKind::Int64 || scratch_stride_ != 0 || exec_ == nullptr || - on_worker_pool_thread() || std::max(rows_offered_, rows) < kIntOwnedMinRows) { - return false; - } - // As `try_owned`: fan-out permission and the worker cap are the - // plan's `partition` phase (src/runtime/PARALLELISM.md); - // `kIntOwnedMinRows` stays as this specialization's admission gate. - if (par_.partition.decline != physical::FanOutDecline::None || - par_.partition.worker_cap < 2) { - return false; - } - auto& pool = process_worker_pool(); - const std::size_t workers = par_.partition.worker_cap; - owned_async_part_count_ = 1; - while (owned_async_part_count_ * 2 <= workers) { - owned_async_part_count_ *= 2; - } - owned_int_partitions_.resize(owned_async_part_count_); - owned_async_group_.emplace(pool.task_group()); - owned_async_hot_mode_ = true; - owned_mode_ = true; - note_partition_fanout(); - } - - const ColumnEntry& agg0 = *agg_entries[0]; - auto job = std::make_unique(); - job->key_column = key_column; - job->sum_column = agg0.column; - if (agg0.validity.has_value()) { - job->sum_validity = *agg0.validity; - } - job->row_base = owned_rows_seen_; - job->rows = rows; - job->part_count = owned_async_part_count_; - auto* const raw_job = job.get(); - owned_async_jobs_.push_back(std::move(job)); - // Engaged since the block above either emplaced it or `owned_async_hot_mode_` - // was already set (the two only ever change together). - if (!owned_async_group_.has_value()) { - invariant_violation("async hot aggregate: task group missing while accepting chunks"); - } - owned_async_group_->submit([raw_job] { process_owned_hot_chunk(*raw_job); }); - owned_rows_seen_ += rows; - return true; - } - - /// Production ownership threshold for the narrow PairIntKey path below, - /// backed by a synthetic row/cardinality sweep (32k/64k/128k/262144 rows - /// x low/high cardinality, 8 cores, 6 interleaved rounds): 32k showed no - /// reliable win (3/6 wins, ~0%), 64k was the first point with a - /// consistent, real one (6/6 wins, -6% to -10%), and 128k/262144 stayed - /// positive. Deliberately NOT `kDefaultPartitionMinRows` (262144, the - /// threshold `try_discover_partitioned` uses): that value was tuned for a - /// different mechanism (discovery only, no fused accumulation, no - /// deferred merge) and is not evidence for where THIS path's overhead - /// breaks even -- q20's own chunks (~150k rows) sit between the two. - static constexpr std::size_t kPairOwnedMinRows = 1U << 16U; // 65536 - - /// Same gate for the single-Int64-key slice. Held at the pair path's value - /// until the sweep below says otherwise -- the mechanism is identical and - /// its break-even has no reason to differ by more than the key's own probe - /// cost, which is lower, not higher. - static constexpr std::size_t kIntOwnedMinRows = 1U << 16U; // 65536 - - /// Production PairIntKey ownership path (TPC-H q20's - /// `by { l_partkey, l_suppkey }` is the motivating shape; validated - /// there at -16.5%, 8/8 paired wins, 8 cores, vs. a q18/Int64 prototype - /// that measured only -7.6%, was never promoted, and has since been - /// removed -- see plans/parallelism-overview.md). Deliberately narrow: - /// - /// - Exactly one aggregate, Sum(Double) or Count. q18 and q20 both only - /// ever exercise one, so nothing measures whether row-wise fusion beats - /// partition-outer/aggregate-outer accumulation once a query carries - /// several -- widen only after that shape is actually benchmarked. - /// - Row-wise fusion only: with exactly one aggregate a second full row - /// scan can only add cost, never locality it does not already have. - /// - No env-var mode selector: this runs whenever eligible, the same way - /// `try_discover_partitioned` has no toggle either. `IBEX_DISABLE_ - /// OWNED_PAIR_AGG=1` is a kill switch for the unusual case that needs - /// one, not a normal control surface. - template - auto try_owned(const KeyAt& key_at, std::size_t rows, std::uint32_t* gids, - const std::vector& agg_entries, Partitions& partitions, - std::size_t min_rows) -> bool { - if (std::getenv("IBEX_DISABLE_OWNED_PAIR_AGG") != nullptr) { - return false; - } - if (!owned_mode_) { - if (n_groups_ > 0 || partitioned_active_) { - return false; - } - if (n_aggs_ != 1) { - return false; - } - if (plan_[0].func != ir::AggFunc::Sum && plan_[0].func != ir::AggFunc::Count) { - return false; - } - if (plan_[0].func == ir::AggFunc::Sum && plan_[0].kind != ExprType::Double) { - return false; - } - if (scratch_stride_ != 0) { - return false; - } - if (exec_ == nullptr || on_worker_pool_thread()) { - return false; - } - // Fan-out permission and the worker cap are the plan's - // (src/runtime/PARALLELISM.md); `decline != None` folds in - // `!exec_->can_fan_out()`. `min_rows` stays here: it is the owned - // strategy's own "is the specialization worth it" gate, lower than - // the radix `partition` floor, and an operator-resolved choice like - // the join's build orientation. - if (par_.partition.decline != physical::FanOutDecline::None || - par_.partition.worker_cap < 2) { - return false; - } - if (std::max(rows_offered_, rows) < min_rows) { - return false; - } - } - - auto& pool = process_worker_pool(); - const std::size_t workers = par_.partition.worker_cap; - std::size_t part_count = 1; - while (part_count * 2 <= workers) { - part_count *= 2; - } - const std::uint64_t part_mask = part_count - 1; - if (partitions.size() < part_count) { - partitions.resize(part_count); - } - note_partition_fanout(); - - // A clustered count key should not pay the partition/scatter/hash - // pipeline once per ROW. Compress contiguous equal-key runs first and - // carry their lengths as count weights. This is exact for arbitrary - // input order: non-contiguous runs still meet in the exact hash - // fallback, while sorted inputs such as TPC-H lineitem reduce the - // expensive part of the pipeline by roughly their mean run length. - // Sample before committing because all-unique keys would only add two - // equality passes and retain the original item count. - // NOLINTNEXTLINE(misc-const-correctness) -- mutated in the int64 instantiation below - [[maybe_unused]] bool compress_runs = false; - if constexpr (std::is_same_v) { - compress_runs = owned_ordered_run_mode_; - if (!compress_runs && !owned_mode_ && plan_[0].func == ir::AggFunc::Count && - rows >= 64 && std::getenv("IBEX_DISABLE_ORDERED_RUN_AGG") == nullptr) { - const std::size_t sampled = std::min(rows, 8192); - std::size_t repeats = 0; - Key previous = key_at(0); - for (std::size_t row = 1; row < sampled; ++row) { - const Key key = key_at(row); - repeats += key == previous ? 1 : 0; - previous = key; - } - compress_runs = repeats * 2 >= sampled - 1; - } - } - - const std::size_t source_ranges = workers; - const std::size_t source_grain = (rows + source_ranges - 1) / source_ranges; - - // Run compression only ever engages for a clustered single-Int64 Count - // (the `if constexpr` above is the only writer of `compress_runs`). - // Two parallel passes, no scratch: pass 1 counts runs per range so the - // per-range output offsets are contiguous, pass 2 emits (key, length) - // straight into `owned_ordered_run_{keys,counts}_`. A third pass to - // dereference anchor rows -- and the `owned_run_rows_/_lengths_` arrays - // it read -- used to sit between them; folding it into pass 2 drops one - // pool barrier per chunk, which is q21's `count() by l_orderkey` hot - // path (~23 chunks, this was 3 barriers each). - if constexpr (std::is_same_v) { - if (compress_runs) { - std::vector run_offsets(source_ranges + 1, 0); - { - auto batch = pool.submit(source_ranges, [&](std::size_t r) { - const std::size_t begin = r * source_grain; - const std::size_t end = std::min(rows, begin + source_grain); - if (begin >= end) { - return; - } - std::size_t runs = 1; - Key previous = key_at(begin); - for (std::size_t row = begin + 1; row < end; ++row) { - const Key key = key_at(row); - runs += key == previous ? 0 : 1; - previous = key; - } - run_offsets[r + 1] = runs; - }); - batch.wait(); - } - for (std::size_t r = 0; r < source_ranges; ++r) { - run_offsets[r + 1] += run_offsets[r]; - } - const std::size_t items = run_offsets.back(); - const std::size_t old_runs = owned_ordered_run_keys_.size(); - owned_ordered_run_keys_.resize(old_runs + items); - owned_ordered_run_counts_.resize(old_runs + items); - { - auto batch = pool.submit(source_ranges, [&](std::size_t r) { - const std::size_t begin = r * source_grain; - const std::size_t end = std::min(rows, begin + source_grain); - if (begin >= end) { - return; - } - std::size_t out = old_runs + run_offsets[r]; - std::size_t anchor = begin; - Key previous = key_at(begin); - for (std::size_t row = begin + 1; row < end; ++row) { - const Key key = key_at(row); - if (key != previous) { - owned_ordered_run_keys_[out] = previous; - owned_ordered_run_counts_[out] = row - anchor; - ++out; - anchor = row; - previous = key; - } - } - owned_ordered_run_keys_[out] = previous; - owned_ordered_run_counts_[out] = end - anchor; - }); - batch.wait(); - } - if (owned_ordered_runs_nondecreasing_) { - const std::size_t from = old_runs == 0 ? 1 : old_runs; - for (std::size_t i = from; i < old_runs + items; ++i) { - if (owned_ordered_run_keys_[i] < owned_ordered_run_keys_[i - 1]) { - owned_ordered_runs_nondecreasing_ = false; - break; - } - } - } - owned_rows_seen_ += rows; - owned_mode_ = true; - owned_ordered_run_mode_ = true; - return true; - } - } - - const std::size_t ranges = workers; - const std::size_t grain = (rows + ranges - 1) / ranges; - part_of_row_.resize(rows); - std::vector counts(ranges * part_count, 0); - { - auto batch = pool.submit(ranges, [&](std::size_t r) { - const std::size_t begin = r * grain; - const std::size_t end = std::min(rows, begin + grain); - std::size_t* row_counts = counts.data() + (r * part_count); - Hash hasher; - for (std::size_t row = begin; row < end; ++row) { - const auto part = static_cast(hasher(key_at(row)) & part_mask); - part_of_row_[row] = part; - ++row_counts[part]; - } - }); - batch.wait(); - } - std::vector offsets(ranges * part_count, 0); - std::vector part_begin(part_count + 1, 0); - { - std::size_t running = 0; - for (std::size_t p = 0; p < part_count; ++p) { - part_begin[p] = running; - for (std::size_t r = 0; r < ranges; ++r) { - offsets[(r * part_count) + p] = running; - running += counts[(r * part_count) + p]; - } - } - part_begin[part_count] = running; - } - scatter_rows_.resize(rows); - { - auto batch = pool.submit(ranges, [&](std::size_t r) { - const std::size_t begin = r * grain; - const std::size_t end = std::min(rows, begin + grain); - std::size_t* cursor = offsets.data() + (r * part_count); - for (std::size_t row = begin; row < end; ++row) { - scatter_rows_[cursor[part_of_row_[row]]++] = row; - } - }); - batch.wait(); - } - - std::vector sum_cols(n_aggs_, nullptr); - std::vector sum_validity(n_aggs_, nullptr); - std::vector is_count(n_aggs_, 0); - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (plan_[a].func == ir::AggFunc::Count) { - is_count[a] = 1; - continue; - } - const ColumnEntry& entry = *agg_entries[a]; - sum_cols[a] = std::get>(*entry.column).data(); - sum_validity[a] = entry.validity.has_value() ? &*entry.validity : nullptr; - } - - const std::uint64_t row_base = owned_rows_seen_; - { - std::atomic cursor{0}; - auto batch = pool.submit(std::min(workers, part_count), [&](std::size_t) { - for (std::size_t p = cursor.fetch_add(1, std::memory_order_relaxed); p < part_count; - p = cursor.fetch_add(1, std::memory_order_relaxed)) { - auto& partition = partitions[p]; - for (std::size_t i = part_begin[p]; i < part_begin[p + 1]; ++i) { - const std::size_t row = scatter_rows_[i]; - const Key key = key_at(row); - auto it = partition.index.find(key); - std::uint32_t local{}; - if (it == partition.index.end()) { - local = static_cast(partition.keys.size()); - partition.index.emplace(key, local); - partition.keys.push_back(key); - partition.first_rows.push_back(row_base + row); - partition.slots.resize((local + 1) * n_aggs_); - } else { - local = it->second; - } - gids[row] = local; - // n_aggs_ == 1 is enforced above -- this is a single - // slot update, not a loop over aggregates. Written as - // one, not unrolled, so a future widening to >1 - // aggregate (once actually measured, per the - // class-level comment) is a small diff here. - AggSlotCore& slot = partition.slots[local]; - if (is_count[0] != 0) { - ++slot.count; - } else if (sum_validity[0] == nullptr || (*sum_validity[0])[row]) { - slot.double_value += sum_cols[0][row]; - slot.mark_present(); - } - } - } - }); - batch.wait(); - } - - owned_rows_seen_ += rows; - owned_mode_ = true; - if (std::getenv("IBEX_AGG_PARTITION_DEBUG") != nullptr) { - ibex::formatting::print( - stderr, "[agg_owned] chunk rows={} part_count={} total_rows={}\n", rows, - partitions.size(), static_cast(owned_rows_seen_)); - } - return true; - } - - /// Walk every `owned_pair_partitions_` entry once, in first-occurrence - /// order (a P-way merge over `first_rows`, run once at final emission), - /// and populate `pair_order_`/`flat_slots_` -- the arrays - /// `build_output_chunk`'s `pair_int_fast_path_` branch already reads. - template - void finalize_owned(Partitions& partitions, const ResizeOrder& resize_order, - const StoreKey& store_key) { - if (owned_finalized_) { - return; - } - owned_finalized_ = true; - const std::size_t part_count = partitions.size(); - std::size_t total = 0; - for (const auto& partition : partitions) { - total += partition.keys.size(); - } - n_groups_ = total; - resize_order(total); - AggSlotCore* fs = flat_slots_.grow_uninitialized(total * n_aggs_).data(); - if (total == 0) { - return; - } - - // K-way merge of the `part_count` partition group-lists by `first_rows` - // into the output at ascending `g`. Every partition's `first_rows` is - // strictly ascending and the values are globally unique (they are row - // indices, and each row belongs to one partition), so this is a stable - // merge over a total order -- byte-identical however the segments below - // are split. - const auto merge_segment = [&](std::vector cur, - const std::vector& stop, std::size_t g) { - for (;;) { - std::size_t best = part_count; - std::uint64_t best_row = std::numeric_limits::max(); - for (std::size_t p = 0; p < part_count; ++p) { - if (cur[p] >= stop[p]) { - continue; - } - const std::uint64_t fr = partitions[p].first_rows[cur[p]]; - if (fr < best_row) { - best_row = fr; - best = p; - } - } - if (best == part_count) { - break; - } - const auto& partition = partitions[best]; - const std::size_t local = cur[best]; - store_key(g, partition.keys[local]); - for (std::size_t a = 0; a < n_aggs_; ++a) { - fs[(g * n_aggs_) + a] = partition.slots[(local * n_aggs_) + a]; - } - ++cur[best]; - ++g; - } - }; - - std::vector part_end(part_count); - for (std::size_t p = 0; p < part_count; ++p) { - part_end[p] = partitions[p].keys.size(); - } - - // For q18's 3M groups the serial merge is ~24M comparisons plus 3M - // slot copies -- tens of ms on the calling thread. Split the OUTPUT - // into per-worker rank ranges via merge-path co-ranking: since every - // `first_rows` value is unique, `sum_p lower_bound(first_rows_p, v)` - // steps by exactly one at each value and so equals any target rank at - // exactly one `v`. Each worker then merges the disjoint input slices - // between two frontiers into its disjoint output slice. - // The `finalize` phase's worker ceiling and fan-out permission are the - // plan's (src/runtime/PARALLELISM.md); `part_count` and `total / 4096` - // stay here -- they need the group count discovery just produced. The - // `1U << 17U` group floor is this merge's own threshold, beside it. - std::size_t workers = 1; - if (exec_ != nullptr && par_.finalize.decline == physical::FanOutDecline::None && - !on_worker_pool_thread() && part_count >= 2 && total >= std::size_t{1} << 17U) { - workers = std::min({par_.finalize.worker_cap, part_count, total / 4096}); - } - - if (workers < 2) { - merge_segment(std::vector(part_count, 0), part_end, 0); - return; - } - note_finalize_fanout(); - - const std::uint64_t hi = owned_rows_seen_ + 1; - const auto frontier = [&](std::size_t rank) { - std::uint64_t lo = 0; - std::uint64_t high = hi; - while (lo < high) { - const std::uint64_t mid = lo + ((high - lo) / 2); - std::size_t sum = 0; - for (std::size_t p = 0; p < part_count; ++p) { - const auto& fr = partitions[p].first_rows; - sum += static_cast(std::lower_bound(fr.begin(), fr.end(), mid) - - fr.begin()); - } - if (sum < rank) { - lo = mid + 1; - } else { - high = mid; - } - } - std::vector off(part_count); - for (std::size_t p = 0; p < part_count; ++p) { - const auto& fr = partitions[p].first_rows; - off[p] = static_cast(std::lower_bound(fr.begin(), fr.end(), lo) - - fr.begin()); - } - return off; - }; - - std::vector> bounds(workers + 1); - bounds.front().assign(part_count, 0); - bounds.back() = part_end; - for (std::size_t w = 1; w < workers; ++w) { - bounds[w] = frontier(w * total / workers); - } - - auto batch = process_worker_pool().submit(workers, [&](std::size_t w) { - std::size_t g = 0; - for (std::size_t p = 0; p < part_count; ++p) { - g += bounds[w][p]; - } - merge_segment(bounds[w], bounds[w + 1], g); - }); - batch.wait(); - } - - /// Dispatch the deferred merge to whichever key the owned run filled. Only - /// one can be non-empty: the gate admits an owned run only before any group - /// exists, so a single operator commits to one key and keeps it. - void finalize_owned_active() { - if (owned_async_hot_mode_) { - finalize_owned_async_hot(); - } else if (owned_ordered_run_mode_) { - finalize_owned_ordered_runs(); - } else if (!owned_pair_partitions_.empty()) { - finalize_owned( - owned_pair_partitions_, [&](std::size_t n) { pair_order_.resize(n); }, - [&](std::size_t g, const PairIntKey& key) { - pair_order_[g] = {static_cast(key.first), - static_cast(key.second)}; - }); - } else if (!owned_int_partitions_.empty()) { - finalize_owned( - owned_int_partitions_, [&](std::size_t n) { int_order_.resize(n); }, - [&](std::size_t g, std::int64_t key) { int_order_[g] = key; }); - } - } - - /// Join the streaming hot-table tasks once, then let one worker own each - /// cold partition for its complete lifetime. There is no chunk-local - /// histogram, scatter, or accumulate barrier: chunks only publish tasks; - /// the two synchronization points here are whole-stream hot completion and - /// whole-stream cold completion. - void finalize_owned_async_hot() { - if (owned_finalized_) { - return; - } - owned_finalized_ = true; - if (!owned_async_group_.has_value()) { - invariant_violation("async hot aggregate join: task group already released"); - } - try { - owned_async_group_->wait(); - } catch (const std::exception& error) { - owned_async_error_ = "async hot aggregate join: " + std::string(error.what()); - return; - } catch (...) { - owned_async_error_ = "async hot aggregate join: non-standard worker exception"; - return; - } - for (const auto& job : owned_async_jobs_) { - if (job->error.has_value()) { - owned_async_error_ = *job->error; - return; - } - } - - auto& partitions = owned_int_partitions_; - const std::size_t part_count = owned_async_part_count_; - if (part_count >= 2) { - note_finalize_fanout(); - } - try { - auto batch = process_worker_pool().submit(part_count, [&](std::size_t p) { - auto& partition = partitions[p]; - std::size_t records = 0; - for (const auto& job : owned_async_jobs_) { - records += job->records_by_partition[p].size(); - } - - // This reserve runs concurrently per owner. Unlike the failed - // calling-thread reserve experiment, it neither serializes the - // partitions nor guesses from total input rows; the exact - // pre-aggregate count is a safe upper bound on distinct keys. - partition.index.reserve(records); - partition.keys.reserve(records); - partition.first_rows.reserve(records); - partition.slots.reserve(records); - - for (const auto& job : owned_async_jobs_) { - for (const auto& record : job->records_by_partition[p]) { - auto it = partition.index.find(record.key); - std::uint32_t local{}; - if (it == partition.index.end()) { - local = static_cast(partition.keys.size()); - partition.index.emplace(record.key, local); - partition.keys.push_back(record.key); - partition.first_rows.push_back(record.first_row); - partition.slots.push_back(record.slot); - } else { - local = it->second; - if (record.slot.present()) { - auto& slot = partition.slots[local]; - slot.double_value += record.slot.double_value; - slot.mark_present(); - } - } - } - } - }); - batch.wait(); - } catch (const std::exception& error) { - owned_async_error_ = "async cold aggregate: " + std::string(error.what()); - return; - } catch (...) { - owned_async_error_ = "async cold aggregate: non-standard worker exception"; - return; - } - - // Pre-aggregate payloads are no longer needed. Release them before the - // output arrays are allocated so peak memory is cold state + output, - // rather than cold state + every streamed record + output. - owned_async_jobs_.clear(); - owned_async_group_.reset(); - - // Reuse the already-parallel first-occurrence merge. `first_rows` in - // every cold partition is ascending because hot records are created at - // their first source row and jobs are consumed in input order. - owned_finalized_ = false; - finalize_owned( - partitions, [&](std::size_t n) { int_order_.resize(n); }, - [&](std::size_t g, std::int64_t key) { int_order_[g] = key; }); - } - - /// A clustered single-Int64 Count is summarized as contiguous runs while - /// chunks arrive. If the complete stream is nondecreasing, adjacent runs - /// are the final groups and no hash table is needed at all. If a later - /// chunk disproves ordering, merge the run summaries through a hash map at - /// emission; this preserves exact first-occurrence semantics without - /// retaining or replaying the input rows. - void finalize_owned_ordered_runs() { - if (owned_finalized_) { - return; - } - owned_finalized_ = true; - if (owned_ordered_run_keys_.empty()) { - n_groups_ = 0; - return; - } - - if (owned_ordered_runs_nondecreasing_) { - const bool ord_timing = std::getenv("IBEX_ORDERED_RUN_TIMING") != nullptr; - const auto ord_t0 = std::chrono::steady_clock::now(); - const std::size_t run_count = owned_ordered_run_keys_.size(); - const std::int64_t* const rk = owned_ordered_run_keys_.data(); - const std::size_t* const rc = owned_ordered_run_counts_.data(); - - // How many workers can help. The build loop below is a segmented - // reduction over `run_count` sorted (key, count) runs: keys are - // nondecreasing so a group boundary is just `rk[i] != rk[i-1]`. - // Each worker owns a contiguous run slice; a key straddling a slice - // boundary has its leading partial count carried back to the group - // the previous worker finished (at most `workers - 1` fixups). - // Ceiling and permission from the plan; `run_count / 8192` and this - // path's own `1U << 16U` run floor stay here (the ordered-run merge - // is a strategy specialization, floor beside its code). - std::size_t workers = 1; - if (exec_ != nullptr && par_.finalize.decline == physical::FanOutDecline::None && - !on_worker_pool_thread() && run_count >= (std::size_t{1} << 16U) && - std::getenv("IBEX_DISABLE_PARALLEL_ORDERED_MERGE") == nullptr) { - workers = std::min({par_.finalize.worker_cap, run_count / 8192}); - } - - std::size_t total = 0; - if (workers < 2) { - total = run_count == 0 ? 0 : 1; - for (std::size_t i = 1; i < run_count; ++i) { - total += rk[i] == rk[i - 1] ? 0 : 1; - } - n_groups_ = total; - int_order_.resize(total); - AggSlotCore* slots = flat_slots_.grow_uninitialized(total).data(); - if (total != 0) { - std::size_t group = 0; - int_order_[0] = rk[0]; - slots[0] = AggSlotCore{}; - slots[0].count = static_cast(rc[0]); - for (std::size_t i = 1; i < run_count; ++i) { - if (rk[i] != int_order_[group]) { - ++group; - int_order_[group] = rk[i]; - slots[group] = AggSlotCore{}; - } - slots[group].count += static_cast(rc[i]); - } - } - } else { - note_finalize_fanout(); - auto& pool = process_worker_pool(); - const std::size_t grain = (run_count + workers - 1) / workers; - std::vector local_groups(workers, 0); - std::vector boundary_open(workers, 0); - { - auto batch = pool.submit(workers, [&](std::size_t w) { - const std::size_t begin = w * grain; - const std::size_t end = std::min(run_count, begin + grain); - if (begin >= end) { - return; - } - std::size_t g = 0; - for (std::size_t i = begin; i < end; ++i) { - if (i == 0 || rk[i] != rk[i - 1]) { - ++g; - } - } - local_groups[w] = g; - boundary_open[w] = (begin > 0 && rk[begin] == rk[begin - 1]) ? 1 : 0; - }); - batch.wait(); - } - std::vector group_base(workers + 1, 0); - for (std::size_t w = 0; w < workers; ++w) { - group_base[w + 1] = group_base[w] + local_groups[w]; - } - total = group_base[workers]; - n_groups_ = total; - int_order_.resize(total); - AggSlotCore* slots = flat_slots_.grow_uninitialized(total).data(); - std::vector carry(workers, 0); - { - auto batch = pool.submit(workers, [&](std::size_t w) { - const std::size_t begin = w * grain; - const std::size_t end = std::min(run_count, begin + grain); - if (begin >= end) { - return; - } - std::size_t i = begin; - if (boundary_open[w] != 0) { - const std::int64_t k0 = rk[begin]; - std::int64_t c = 0; - while (i < end && rk[i] == k0) { - c += static_cast(rc[i]); - ++i; - } - carry[w] = c; - } - std::size_t g = group_base[w]; - while (i < end) { - const std::int64_t k = rk[i]; - std::int64_t c = 0; - while (i < end && rk[i] == k) { - c += static_cast(rc[i]); - ++i; - } - int_order_[g] = k; - slots[g] = AggSlotCore{}; - slots[g].count = c; - ++g; - } - }); - batch.wait(); - } - for (std::size_t w = 0; w < workers; ++w) { - if (boundary_open[w] != 0) { - slots[group_base[w] - 1].count += carry[w]; - } - } - } - - if (ord_timing) { - const auto ms = std::chrono::duration( - std::chrono::steady_clock::now() - ord_t0) - .count(); - ibex::formatting::print(stderr, - "[ord_run] finalize nondecreasing runs={} groups={} " - "workers={} {}ms\n", - run_count, total, workers, ms); - } - return; - } - - robin_hood::unordered_flat_map index; - std::vector counts; - for (std::size_t i = 0; i < owned_ordered_run_keys_.size(); ++i) { - const std::int64_t key = owned_ordered_run_keys_[i]; - auto it = index.find(key); - std::uint32_t group{}; - if (it == index.end()) { - group = static_cast(int_order_.size()); - index.emplace(key, group); - int_order_.push_back(key); - counts.push_back(0); - } else { - group = it->second; - } - counts[group] += static_cast(owned_ordered_run_counts_[i]); - } - n_groups_ = int_order_.size(); - AggSlotCore* slots = flat_slots_.grow_uninitialized(n_groups_).data(); - for (std::size_t group = 0; group < n_groups_; ++group) { - slots[group] = AggSlotCore{}; - slots[group].count = counts[group]; - } - } - - // Two fixed-width-integer keys, grouped as one composite. Mirrors - // process_rows_int exactly, packing (key_a, key_b) into a two-word key so - // a single hash probe replaces the generic path's per-key Key comparison. - auto process_rows_int_pair(const std::vector& group_entries, - const std::vector& agg_entries, std::size_t rows) - -> std::optional { - // Bind the key column's buffer once, the way `process_rows_int` does. - // Reading it through `std::get` per row costs a variant index check per - // key per row and re-derives the pointer every time, which on 8M rows - // over two keys was most of this loop. - struct RawKeyReader { - const std::int64_t* i64 = nullptr; - const Date* dates = nullptr; - const Timestamp* stamps = nullptr; - const Column::code_type* codes = nullptr; - IntKeyKind kind = IntKeyKind::Int64; - - [[nodiscard]] auto operator()(std::size_t row) const -> std::int64_t { - switch (kind) { - case IntKeyKind::Int64: - return i64[row]; - case IntKeyKind::Date: - return dates[row].days; - case IntKeyKind::Ts: - return stamps[row].nanos; - case IntKeyKind::Cat: - return codes[row]; - } - return 0; - } - }; - const auto bind_reader = [](const ColumnValue& col, IntKeyKind kind) -> RawKeyReader { - RawKeyReader reader; - reader.kind = kind; - switch (kind) { - case IntKeyKind::Int64: - reader.i64 = std::get>(col).data(); - break; - case IntKeyKind::Date: - reader.dates = std::get>(col).data(); - break; - case IntKeyKind::Ts: - reader.stamps = std::get>(col).data(); - break; - case IntKeyKind::Cat: - reader.codes = std::get>(col).codes_data(); - break; - } - return reader; - }; - const auto key_a_at = bind_reader(*group_entries[0]->column, int_key_kind_); - const auto key_b_at = bind_reader(*group_entries[1]->column, int_key_kind_b_); - const auto pack = [](std::int64_t a, std::int64_t b) -> PairIntKey { - return {.first = static_cast(a), - .second = static_cast(b)}; - }; - - gids_buf_.resize(rows); - auto* gids = gids_buf_.data(); - - // A Categorical code and a Date are both 32 bits wide, so when both - // keys are one of those the composite fits in 64 bits exactly and can - // be probed in the same flat int map the single-int path uses. That is - // the common shape of `by { symbol, day }`, and it halves the key - // width, the hash and the stored entry against the 128-bit form. - // Both key domains are 32 bits wide here, so the composite is exact. - const auto pack_u64 = [](std::int64_t a, std::int64_t b) -> std::int64_t { - return static_cast( - (static_cast(static_cast(a)) << 32U) | - static_cast(static_cast(b))); - }; - if (pair_packs_u64_) { - // Both key domains are narrow enough to enumerate: a Categorical - // spans its dictionary, and a Date column's span is measured. When - // the product fits, index a flat cell -> gid array and the per-row - // hash disappears entirely -- the same trick, and the same reason, - // as the all-Categorical Cartesian path. `by { day }` over 4 - // distinct days was costing 34ms on 8M rows purely in hash probes. - if (try_process_rows_pair_dense(key_a_at, key_b_at, group_entries, agg_entries, rows)) { - return std::nullopt; - } - // Discovery across workers, for the case the dense array cannot - // hold: the cell budget is a product, so a wide symbol domain times - // a wide day domain overflows it long before either alone is - // remarkable, and the u64 key that falls out is the CHEAPEST key in - // this file to partition. Until now this branch returned before ever - // reaching `try_discover_partitioned` — a `by { symbol, day }` over - // 5000 symbols and 1000 days ran wholly serially. - // - // Only while the dense path has never run. Dense numbers groups in - // its own array and rebuilds that array from `pair_order_`, so it - // can safely take over from partitioned discovery; the reverse is - // not true, because the partitions would not know the groups dense - // had already numbered and would issue second ids for them. - if (!pair_dense_active_ && - try_discover_partitioned>( - [&](std::size_t row) { return pack_u64(key_a_at(row), key_b_at(row)); }, rows, - gids, int_partitions_, [&](std::size_t n) { pair_order_.resize(n); }, - [&](std::int64_t, std::uint32_t gid, std::size_t row) { - // From the row, not by unpacking: `pair_order_` holds the - // reader's own values, and the pack truncates to 32 bits. - pair_order_[gid] = {key_a_at(row), key_b_at(row)}; - }, - kDefaultPartitionMinRows, - [&](std::uint32_t gid) { - // The pack is a pure function of the pair, so a group's - // key is recoverable even though the pack is lossy. - return pack_u64(pair_order_[gid].first, pair_order_[gid].second); - })) { - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - // Falling here with groups already numbered means the dense path ran - // on an earlier chunk and this chunk's domains overflowed its budget. - // `int_index_` has no record of those groups, so without this it - // would issue a second id for each and the output would carry two - // rows per key. `pair_order_` is the pair path's source of truth — - // this is the same rebuild dense itself does when its bounds move. - if (int_index_.size() < pair_order_.size()) { - int_index_.reserve(pair_order_.size()); - for (std::size_t g = 0; g < pair_order_.size(); ++g) { - int_index_.emplace(pack_u64(pair_order_[g].first, pair_order_[g].second), - static_cast(g)); - } - } - - std::int64_t prev_packed = 0; - std::uint32_t prev_gid_u64 = std::numeric_limits::max(); - bool have_prev_u64 = false; - for (std::size_t row = 0; row < rows; ++row) { - const std::int64_t a = key_a_at(row); - const std::int64_t b = key_b_at(row); - const std::int64_t key = pack_u64(a, b); - std::uint32_t gid{}; - if (have_prev_u64 && key == prev_packed) { - gid = prev_gid_u64; - } else { - auto it = int_index_.find(key); - if (it == int_index_.end()) { - gid = static_cast(n_groups_); - int_index_.emplace(key, gid); - pair_order_.emplace_back(a, b); - ++n_groups_; - size_group_arrays(); - } else { - gid = it->second; - } - prev_packed = key; - prev_gid_u64 = gid; - have_prev_u64 = true; - } - gids[row] = gid; - } - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - if (try_owned( - [&](std::size_t row) { return pack(key_a_at(row), key_b_at(row)); }, rows, gids, - agg_entries, owned_pair_partitions_, kPairOwnedMinRows)) { - return std::nullopt; - } - - if (try_discover_partitioned( - [&](std::size_t row) { return pack(key_a_at(row), key_b_at(row)); }, rows, gids, - pair_partitions_, [&](std::size_t n) { pair_order_.resize(n); }, - [&](const PairIntKey& key, std::uint32_t gid, std::size_t) { - pair_order_[gid] = {static_cast(key.first), - static_cast(key.second)}; - }, - kDefaultPartitionMinRows, - [&](std::uint32_t gid) { - return pack(pair_order_[gid].first, pair_order_[gid].second); - })) { - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - PairIntKey prev_key{}; - std::uint32_t prev_gid = std::numeric_limits::max(); - bool have_prev = false; - for (std::size_t row = 0; row < rows; ++row) { - const std::int64_t a = key_a_at(row); - const std::int64_t b = key_b_at(row); - const PairIntKey key = pack(a, b); - std::uint32_t gid{}; - if (have_prev && key == prev_key) { - gid = prev_gid; - } else { - auto it = pair_index_.find(key); - if (it == pair_index_.end()) { - gid = static_cast(n_groups_); - pair_index_.emplace(key, gid); - pair_order_.emplace_back(a, b); - ++n_groups_; - size_group_arrays(); - } else { - gid = it->second; - } - prev_key = key; - prev_gid = gid; - have_prev = true; - } - gids[row] = gid; - } - - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - /// Bounds of one key column over a chunk, as the dense cell numbering needs - /// them. A Categorical answers from its dictionary without reading a row; - /// anything else is measured. - template - auto key_bounds(const Reader& read, const ColumnValue& col, IntKeyKind kind, std::size_t rows) - -> std::pair { - if (kind == IntKeyKind::Cat) { - const auto size = std::get>(col).dictionary().size(); - return {0, size == 0 ? 0 : static_cast(size) - 1}; - } - std::int64_t lo = read(0); - std::int64_t hi = lo; - for (std::size_t row = 1; row < rows; ++row) { - const std::int64_t v = read(row); - lo = std::min(lo, v); - hi = std::max(hi, v); - } - return {lo, hi}; - } - - /// Group a packed 32-bit key pair through a flat cell array. Returns false - /// when the key domains are too large to enumerate, leaving the caller on - /// the hash path. - /// - /// The cell numbering is a function of the bounds, so a later chunk that - /// widens them invalidates every cell already handed out. That is handled - /// the way the multi-key Categorical path handles a stride change: widen to - /// the union and rebuild the array from `pair_order_`, which holds each - /// group's key pair. Group ids themselves never move. - template - auto try_process_rows_pair_dense(const ReaderA& key_a_at, const ReaderB& key_b_at, - // One caller - const std::vector& group_entries, - const std::vector& agg_entries, - std::size_t rows) -> bool { - if (rows == 0) { - return false; - } - const auto [a_lo, a_hi] = - key_bounds(key_a_at, *group_entries[0]->column, int_key_kind_, rows); - const auto [b_lo, b_hi] = - key_bounds(key_b_at, *group_entries[1]->column, int_key_kind_b_, rows); - - std::int64_t a_min = a_lo; - std::int64_t b_min = b_lo; - std::int64_t a_max = a_hi; - std::int64_t b_max = b_hi; - if (pair_dense_active_) { - a_min = std::min(a_min, pair_dense_a_min_); - b_min = std::min(b_min, pair_dense_b_min_); - a_max = std::max(a_max, pair_dense_a_max_); - b_max = std::max(b_max, pair_dense_b_max_); - } - - // Spans are computed in unsigned arithmetic so a domain that legitimately - // straddles zero cannot overflow the subtraction. - const auto a_span = static_cast(a_max - a_min) + 1; - const auto b_span = static_cast(b_max - b_min) + 1; - if (b_span != 0 && a_span > kDenseCellLimit / b_span) { - return false; // product overflows the dense budget - } - const std::uint64_t cells = a_span * b_span; - if (cells > kDenseCellLimit) { - return false; - } - - const bool bounds_changed = !pair_dense_active_ || a_min != pair_dense_a_min_ || - b_min != pair_dense_b_min_ || b_span != pair_dense_b_span_; - if (bounds_changed) { - pair_dense_gid_.assign(static_cast(cells), kNoGid); - for (std::size_t g = 0; g < pair_order_.size(); ++g) { - const auto cell = - (static_cast(pair_order_[g].first - a_min) * b_span) + - static_cast(pair_order_[g].second - b_min); - pair_dense_gid_[static_cast(cell)] = static_cast(g); - } - pair_dense_a_min_ = a_min; - pair_dense_b_min_ = b_min; - pair_dense_a_max_ = a_max; - pair_dense_b_max_ = b_max; - pair_dense_b_span_ = b_span; - pair_dense_active_ = true; - } else if (pair_dense_gid_.size() < cells) { - pair_dense_gid_.resize(static_cast(cells), kNoGid); - pair_dense_a_max_ = a_max; - } - - gids_buf_.resize(rows); - auto* gids = gids_buf_.data(); - std::uint32_t* dense = pair_dense_gid_.data(); - for (std::size_t row = 0; row < rows; ++row) { - const std::int64_t a = key_a_at(row); - const std::int64_t b = key_b_at(row); - const auto cell = (static_cast(a - a_min) * b_span) + - static_cast(b - b_min); - std::uint32_t gid = dense[cell]; - if (gid == kNoGid) { - gid = static_cast(n_groups_); - dense[cell] = gid; - pair_order_.emplace_back(a, b); - ++n_groups_; - size_group_arrays(); - dense = pair_dense_gid_.data(); - } - gids[row] = gid; - } - - accumulate_gids(gids, agg_entries, rows); - return true; - } - - /// Slot-indexed boxed value, grown on first use. `slot_index` is the same - /// `gid * n_aggs_ + agg_i` that indexes `flat_slots_`. - auto text_at(std::size_t slot_index) -> ScalarValue& { - if (text_store_.size() < flat_slots_.size()) { - text_store_.resize(flat_slots_.size()); - } - return text_store_[slot_index]; - } - - /// Scratch for one (group, aggregate). Only valid when that aggregate - /// declared scratch_doubles > 0. - [[nodiscard]] auto scratch_for(std::size_t gid, std::size_t agg_i) -> double* { - return scratch_.data() + (gid * scratch_stride_) + scratch_offset_[agg_i]; - } - - /// Size every per-group array to `n_groups_`. THE ONLY PLACE THAT RESIZES - /// THEM — the grouping fast paths used to call `flat_slots_.resize()` - /// directly, and adding a second per-group array (scratch) to just one of - /// those call sites left the others reading a null pointer. - /// One hash partition's share of the group index. Partitions are disjoint by - /// construction — a key's partition is a function of its hash — so a worker - /// owning a partition owns every row and every group in it, and needs no - /// lock and no merge against the others. - /// `Eq` is spelled out so a transparent hash/equal pair can be used: the - /// string path stores owning `std::string` keys but probes with - /// `std::string_view`, and only pays the copy on a genuinely new group — - /// exactly what the serial `str_index_` does. - template > - struct KeyPartition { - robin_hood::unordered_flat_map index; - /// This partition's groups, in the order they were first seen — which, - /// because rows are scattered in row order, is ascending by first row. - /// That is what makes the final ordering a merge of already-sorted - /// lists rather than a sort. - std::vector gids; - std::vector first_rows; - std::vector keys; - /// How many of `keys` have already been written back to the caller's - /// gid-indexed key vector; the rest were added by the current chunk. - std::size_t stored = 0; - }; - - /// Discover groups across workers by hash-partitioning the rows. - /// - /// **Group DISCOVERY is the serial half of a high-cardinality group-by, and - /// it is the half that could not be threaded before.** `accumulate_gids` - /// already fans out the summing, but only once gids exist, and its gate - /// declines exactly when groups are numerous — `morsels * n_groups > rows/4` - /// — because merging per-morsel partial tables then costs more than the scan - /// it saved. PDS-H q20 (543k groups) and q13 (150k) both land there and run - /// wholly serially. - /// - /// Partitioning removes the merge instead of paying it. Each key belongs to - /// exactly one partition, so per-partition tables never have to be - /// reconciled: there is no partial to combine, only a concatenation. It also - /// shrinks each table to a P-th of the rows, which is most of the win at - /// this cardinality — the serial probe is cache-miss bound on a table far - /// larger than L2. - /// - /// Returns false when the shape does not justify it and the caller should - /// run its own serial loop. - /// - /// **Ordering.** Ibex reports groups in first-occurrence order, and gids - /// here are handed out by an atomic, so gid order is a race. The order is - /// recovered from data instead: every group records the row it was first - /// seen at, and `build_output_chunk` walks the groups by that. Rows scatter - /// into partitions in row order, so each partition's list is already - /// ascending and the global order is a P-way merge, not a sort. - /// - /// **Determinism.** A group's rows all live in one partition and are visited - /// in row order, so each group's values accumulate in exactly the order the - /// serial path would use. The output is byte-identical, not merely - /// equivalent — including the float sums. - /// Passed as `key_of_group` by a caller that cannot reconstruct a - /// partition key from a group id, which is what decides whether this path - /// may start part-way through a stream. Only the packed path is in that - /// position: its key is built from a ROW and is not invertible. - struct NoGroupKeys {}; - - template , typename KeyAt, - typename ResizeKeys, typename StoreKey, typename KeyOfGroup = NoGroupKeys> - auto try_discover_partitioned(const KeyAt& key_at, std::size_t rows, std::uint32_t* gids, - std::vector>& partitions, - const ResizeKeys& resize_keys, const StoreKey& store_key, - std::size_t min_rows = kDefaultPartitionMinRows, - const KeyOfGroup& key_of_group = {}) -> bool { - // Below `min_rows` the partition and scatter passes cost more than the - // serial probe they replace. High cardinality is not checkable up front - // — it is what discovery is about to find out — so row count is the only - // gate available, and a low-cardinality run of this size still wins from - // the smaller per-partition tables. - // - // It is a parameter because the break-even is a property of the KEY, not - // of partitioning: the serial probe a packed key replaces is far more - // expensive per row than the one an int key replaces, so it pays off - // sooner. Callers that do not pass it keep the original threshold. - // - // Once this path HAS run, every later chunk must take it too, however - // small. The groups it discovered live in `partitions`, and the serial - // loops probe `int_index_` / `str_index_`, which this path never - // populates — so a small trailing chunk falling back would not find the - // existing groups and would allocate second ids for them. The row gate - // therefore only guards the first use. - // - // The gate counts every row this operator has been OFFERED, not the - // rows in this call. They were the same number while a source produced - // one chunk; once it produces six, a per-call gate sees a sixth of the - // input and declines on a query that plainly qualifies. PDS-H q20 is - // exactly that: 909k rows over 543k groups, which activated this path - // as one chunk and lost it entirely as six, taking the aggregate from - // 50ms to 79ms and the query +23%. The threshold itself is unchanged — - // lowering it is a measured dead end, because the break-even is set by - // group CARDINALITY and a low-cardinality run of this size loses. - constexpr bool can_seed = !std::is_same_v; - // The `partition` phase's worker cap and fan-out permission come from - // the plan (src/runtime/PARALLELISM.md); the operator keeps only the two - // checks it alone can make -- is it nested, did this operator's input so - // far clear the floor. `decline != None` folds in `!exec_->can_fan_out()` - // (the plan resolves `SingleCore` from it). `min_rows` is still the - // operator's: it is the radix strategy's own admission gate, stricter - // than `try_owned`'s, and lives beside the constant it names. - if (exec_ == nullptr || on_worker_pool_thread()) { - return false; - } - if (par_.partition.decline != physical::FanOutDecline::None || - par_.partition.worker_cap < 2) { - return false; - } - if (!partitioned_active_) { - if (std::max(rows_offered_, rows) < min_rows) { - return false; - } - // Starting part-way through means groups already exist, and they - // live in the serial index this path neither reads nor writes. - // They have to be moved across (below) or they would be issued - // second ids; a caller that cannot hand back their keys cannot - // start late at all. - if (n_groups_ > 0 && !can_seed) { - return false; - } - } - auto& pool = process_worker_pool(); - const std::size_t workers = par_.partition.worker_cap; - std::size_t part_count = 1; - while (part_count * 2 <= workers) { - part_count *= 2; // a power of two, so the partition is a mask - } - note_partition_fanout(); - const std::uint64_t part_mask = part_count - 1; - if (partitions.size() < part_count) { - partitions.resize(part_count); - } - - // Adopt the groups the serial path already discovered, so this path can - // start on any chunk rather than only the first. Each keeps its - // existing global id, and `stored` is set past them all: the ordering - // merge below only visits entries added by the current chunk, so their - // `first_rows` are never read and the ids handed out here continue - // after them — which is the same invariant that lets one partitioned - // chunk follow another. - if constexpr (can_seed) { - if (!partitioned_active_ && n_groups_ > 0) { - Hash hasher; - for (std::uint32_t gid = 0; gid < static_cast(n_groups_); ++gid) { - auto key = key_of_group(gid); - auto& partition = partitions[static_cast(hasher(key) & part_mask)]; - partition.index.emplace(Key(key), - static_cast(partition.gids.size())); - partition.gids.push_back(gid); - partition.first_rows.push_back(0); - partition.keys.emplace_back(key); - } - for (auto& partition : partitions) { - partition.stored = partition.gids.size(); - } - } - } - - // Pass 1: partition of every row, and a per-range histogram. Ranges are - // contiguous so that the scatter below keeps rows in row order within a - // partition, which is what the ordering argument above depends on. - const std::size_t ranges = workers; - const std::size_t grain = (rows + ranges - 1) / ranges; - part_of_row_.resize(rows); - std::vector counts(ranges * part_count, 0); - { - auto batch = pool.submit(ranges, [&](std::size_t r) { - const std::size_t begin = r * grain; - const std::size_t end = std::min(rows, begin + grain); - std::size_t* row_counts = counts.data() + (r * part_count); - Hash hasher; - for (std::size_t row = begin; row < end; ++row) { - const auto part = static_cast(hasher(key_at(row)) & part_mask); - part_of_row_[row] = part; - ++row_counts[part]; - } - }); - batch.wait(); - } - - // Exclusive prefix sum, partition-major then range-major, so each range - // writes its own slice of each partition without touching a shared - // cursor. - std::vector offsets(ranges * part_count, 0); - std::vector part_begin(part_count + 1, 0); - { - std::size_t running = 0; - for (std::size_t p = 0; p < part_count; ++p) { - part_begin[p] = running; - for (std::size_t r = 0; r < ranges; ++r) { - offsets[(r * part_count) + p] = running; - running += counts[(r * part_count) + p]; - } - } - part_begin[part_count] = running; - } - - // Pass 2: scatter row indices into their partition's slice. - scatter_rows_.resize(rows); - { - auto batch = pool.submit(ranges, [&](std::size_t r) { - const std::size_t begin = r * grain; - const std::size_t end = std::min(rows, begin + grain); - std::size_t* cursor = offsets.data() + (r * part_count); - for (std::size_t row = begin; row < end; ++row) { - scatter_rows_[cursor[part_of_row_[row]]++] = row; - } - }); - batch.wait(); - } - - // Pass 3: each worker owns whole partitions. Ids assigned here are - // partition-LOCAL — a plain counter, no atomic — because a global id - // cannot be handed out in first-occurrence order until every partition - // has been seen. `gids[row]` therefore holds a local id until pass 4. - const std::uint64_t row_base = rows_seen_; - { - std::atomic cursor{0}; - auto batch = pool.submit(std::min(workers, part_count), [&](std::size_t) { - for (std::size_t p = cursor.fetch_add(1, std::memory_order_relaxed); p < part_count; - p = cursor.fetch_add(1, std::memory_order_relaxed)) { - auto& partition = partitions[p]; - for (std::size_t i = part_begin[p]; i < part_begin[p + 1]; ++i) { - const std::size_t row = scatter_rows_[i]; - // `auto`, not `Key`: the probe type may be a view onto - // the key column (strings), and materializing an owning - // key per ROW rather than per GROUP is the whole cost - // this path exists to avoid. - const auto key = key_at(row); - auto it = partition.index.find(key); - std::uint32_t local{}; - if (it == partition.index.end()) { - local = static_cast(partition.gids.size()); - partition.index.emplace(Key(key), local); - partition.gids.push_back(0); // filled below, in order - partition.first_rows.push_back(row_base + row); - partition.keys.emplace_back(key); - } else { - local = it->second; - } - gids[row] = local; - } - } - }); - batch.wait(); - } - - // Number this chunk's new groups in first-occurrence order, so that gid - // order IS that order and nothing downstream has to compensate. - // - // Emitting in discovery order and permuting at the end was tried first - // and is a trap: the permutation turns the emit's sequential walk of the - // slot array into a random gather, which on q18's 1.5m-group aggregate - // cost more than the parallel discovery saved (+25% median). Paying one - // ordered pass here instead keeps every later pass sequential. - // - // Each partition's new entries are already ascending by first row, so - // this is a P-way merge. Groups carried over from earlier chunks keep - // their ids: a group first seen in a later chunk necessarily has a later - // first row, so appending after them preserves the global order. - // - // Rescanning every partition per group is O(groups x partitions), and - // replacing it with the textbook heap (replace-top, one sift per group, - // O(groups log partitions)) is a MEASURED DEAD END: q18's merge went - // 25.6ms -> 27.1ms and q20's 7.9ms -> 13.1ms, suite +0.15% over 12 - // interleaved rounds. P is `part_count`, a power of two capped by the - // worker count -- 8 here. Eight predictable compares over an array that - // never leaves L1 beat three sift levels of data-dependent branching and - // struct moves. A heap would need dozens of runs before it paid. - const std::size_t base = n_groups_; - { - std::vector cursors(part_count); - for (std::size_t p = 0; p < part_count; ++p) { - cursors[p] = partitions[p].stored; - } - std::uint32_t next = static_cast(base); - while (true) { - std::size_t best = part_count; - std::uint64_t best_row = std::numeric_limits::max(); - for (std::size_t p = 0; p < part_count; ++p) { - if (cursors[p] >= partitions[p].first_rows.size()) { - continue; - } - if (partitions[p].first_rows[cursors[p]] < best_row) { - best_row = partitions[p].first_rows[cursors[p]]; - best = p; - } - } - if (best == part_count) { - break; - } - partitions[best].gids[cursors[best]] = next++; - ++cursors[best]; - } - n_groups_ = next; - } - size_group_arrays(); - // The caller's gid-indexed key vector has to cover the ids just handed - // out before any of them is written back. - resize_keys(n_groups_); - for (auto& partition : partitions) { - for (std::size_t i = partition.stored; i < partition.gids.size(); ++i) { - // The third argument is the group's first row WITHIN THIS - // CHUNK. Only entries from `stored` on are visited, and those - // are exactly the groups this call discovered, so their first - // row is always local and `row_base` recovers it. A packed key - // is not invertible on its own — the packed path uses this row - // to read the original column values back for the output. - store_key(partition.keys[i], partition.gids[i], - static_cast(partition.first_rows[i] - row_base)); - } - partition.stored = partition.gids.size(); - } - - // Pass 4: local id -> global gid. The partition a row belongs to is - // already recorded, so this is a lookup, not a re-probe. - { - auto batch = pool.submit(ranges, [&](std::size_t r) { - const std::size_t begin = r * grain; - const std::size_t end = std::min(rows, begin + grain); - for (std::size_t row = begin; row < end; ++row) { - gids[row] = partitions[part_of_row_[row]].gids[gids[row]]; - } - }); - batch.wait(); - } - rows_seen_ += rows; - partitioned_active_ = true; - return true; - } - - void size_group_arrays() { - auto tail = flat_slots_.grow_uninitialized(n_groups_ * n_aggs_); - if (!fill_slots_parallel(tail)) { - SlotArray::fill_default(tail); - } - if (scratch_stride_ != 0) { - scratch_.resize(n_groups_ * scratch_stride_, 0.0); - } - } - - /// Zero a freshly grown slot tail across workers. Returns false when the - /// tail is too small to be worth a batch — which is every call from - /// `alloc_group`, where the tail is one slot — and the caller fills it - /// serially. - /// - /// Worth threading at all only because the cost is page faults rather than - /// bytes: the kernel materializes a page per fault, and eight threads - /// faulting disjoint pages fault in parallel. Eight threads buy about 1.3x, - /// not 8x — the fault path serializes on the kernel's own locks — so this is - /// a small win, not a lever. Sizing it against the rest of q18's aggregate - /// (3M groups): 80ms parallel discovery probe, 44ms this fill, 26ms serial - /// first-occurrence merge, 12ms key-array growth, 10ms accumulate. It is the - /// largest SERIAL block, which is why it is threaded first, and it is still - /// only a sixth of the operator. - auto fill_slots_parallel(std::span tail) -> bool { - // Sized so the batch (a submit plus a join) stays small against the - // work. Below a few megabytes the serial memset is already - // bandwidth-bound and has no faults left to hide. - constexpr std::size_t kMinTailBytes = 4UL << 20; - if (tail.size() * sizeof(AggSlotCore) < kMinTailBytes) { - return false; - } - if (exec_ == nullptr || !exec_->can_fan_out() || on_worker_pool_thread()) { - return false; - } - auto& pool = process_worker_pool(); - const std::size_t threads = std::min(std::size_t{16}, exec_->compute_budget()); - if (threads < 2) { - return false; - } - const std::size_t grain = (tail.size() + threads - 1) / threads; - auto batch = pool.submit(threads, [&](std::size_t t) { - const std::size_t begin = t * grain; - if (begin >= tail.size()) { - return; - } - const std::size_t end = std::min(tail.size(), begin + grain); - SlotArray::fill_default(tail.subspan(begin, end - begin)); - }); - batch.wait(); - return true; - } - - auto alloc_group() -> std::uint32_t { - auto gid = static_cast(n_groups_); - ++n_groups_; - size_group_arrays(); - return gid; - } - - /// Seed `group_order_`/`key_index_` (the generic path's state) with `n` - /// groups a fast path already discovered, in the same first-seen order - /// the fast path used -- so gid `i` here matches the gid `flat_slots_`/ - /// `scratch_` already hold data for at index `i`. `key_at` builds the - /// full `Key` (every fast path here stores raw values only, never a - /// null, so every migrated `Key` has an empty null mask). - /// - /// Hashing goes through `hash_key_value`, which is defined to agree with - /// `hash_key_row` on every value both can express -- the invariant this - /// whole migration rests on: a later chunk's row-based probe and a - /// migrated group's stored hash must land the same value in the same - /// slot. `KeyRowIndex::rehash` reproduces the exact open-address - /// placement `find_or_insert` would have made one row at a time, so a - /// batch reseed and an incremental build agree on where every group ends - /// up. - template - void seed_generic_index_from_keys(std::size_t n, const KeyAt& key_at) { - group_order_.reserve(group_order_.size() + n); - key_index_.hashes.reserve(key_index_.hashes.size() + n); - for (std::size_t i = 0; i < n; ++i) { - Key key = key_at(i); - key_index_.hashes.push_back(hash_key_value(key)); - group_order_.push_back(std::move(key)); - } - std::size_t capacity = 1024; - while (capacity * 7 < key_index_.hashes.size() * 10) { - capacity *= 2; - } - key_index_.rehash(capacity); - } - - /// A single `int_key_kind_`-typed raw value, as a `ScalarValue` matching - /// what `push_key_value` would have built for the equivalent column. - static auto scalar_of_int_key(IntKeyKind kind, std::int64_t raw) -> ScalarValue { - switch (kind) { - case IntKeyKind::Date: - return Date{.days = static_cast(raw)}; - case IntKeyKind::Ts: - return Timestamp{.nanos = raw}; - case IntKeyKind::Int64: - case IntKeyKind::Cat: - break; - } - return raw; - } - - /// Fold the single-int fast path's raw values (int64 / Date / Timestamp, - /// as `process_rows_int` stores them) into the generic grouping path when - /// a later chunk brings a validity bitmap the fast path cannot express. - /// The accumulated slots stay put -- only the key->gid lookup is rebuilt. - void migrate_int_fast_path_to_generic() { - seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { - Key key; - key.values.push_back(scalar_of_int_key(int_key_kind_, int_order_[i])); - return key; - }); - int_fast_path_ = false; - } - - /// Same migration as `migrate_int_fast_path_to_generic`, for the - /// single-string fast path's `str_order_`. - void migrate_str_fast_path_to_generic() { - seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { - Key key; - key.values.push_back(str_order_[i]); - return key; - }); - str_fast_path_ = false; - } - - /// Fold the categorical fast path -- single-key (`cat_order_`, code == - /// dictionary index) or multi-key (`multi_cat_codes_flat_`, `n_keys` - /// codes per group) -- into the generic grouping path. A code only means - /// something against ITS column's dictionary, which `group_templates_` - /// still holds (the empty `make_empty_like` template built at - /// `initialized_` time shares the dictionary every chunk's column uses), - /// so decoding a migrated group's code to the same string `push_key_value` - /// would have read off the live column is just a dictionary lookup. - void migrate_cat_fast_path_to_generic(std::size_t n_keys) { - const auto decode = [&](std::size_t c, Column::code_type code) { - return std::get>(group_templates_[c]) - .dictionary()[static_cast(code)]; - }; - if (n_keys == 1) { - seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { - Key key; - key.values.push_back(std::string(decode(0, cat_order_[i]))); - return key; - }); - } else { - seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { - Key key; - key.values.reserve(n_keys); - for (std::size_t c = 0; c < n_keys; ++c) { - key.values.push_back( - std::string(decode(c, multi_cat_codes_flat_[(i * n_keys) + c]))); - } - return key; - }); - } - cat_fast_path_ = false; - } - - /// Fold the paired-int fast path's raw values (`pair_order_`, one - /// `int64`/`Date`/`Timestamp`/categorical-code pair per group) into the - /// generic grouping path. `int_key_kind_`/`int_key_kind_b_` name each - /// column's type; a `Cat` column's code decodes through the matching - /// `group_templates_` entry exactly as the single/multi categorical - /// migration does. - void migrate_pair_int_fast_path_to_generic() { - const auto scalar_at = [&](IntKeyKind kind, std::size_t col_index, - std::int64_t raw) -> ScalarValue { - if (kind == IntKeyKind::Cat) { - const auto& dict = - std::get>(group_templates_[col_index]).dictionary(); - return std::string(dict[static_cast(raw)]); - } - return scalar_of_int_key(kind, raw); - }; - seed_generic_index_from_keys(n_groups_, [&](std::size_t i) { - Key key; - key.values.reserve(2); - key.values.push_back(scalar_at(int_key_kind_, 0, pair_order_[i].first)); - key.values.push_back(scalar_at(int_key_kind_b_, 1, pair_order_[i].second)); - return key; - }); - pair_int_fast_path_ = false; - } - - /// Fold the packed (3+ key) fast path into the generic grouping path. - /// - /// Unlike every other fast path here, this one needs no decode at all: - /// `process_rows_packed` already builds a full boxed `Key` per group into - /// `group_order_` from the ROW (see its comment -- the packed word is - /// used only for the FAST lookup, never as the group's stored identity), - /// so `group_order_` is already exactly what the generic path expects. - /// Only `key_index_`, which hashes packed words rather than `Key`s, needs - /// rebuilding from what is already there. - void migrate_packed_fast_path_to_generic() { - key_index_.hashes.clear(); - key_index_.hashes.reserve(group_order_.size()); - for (const auto& key : group_order_) { - key_index_.hashes.push_back(hash_key_value(key)); - } - std::size_t capacity = 1024; - while (capacity * 7 < key_index_.hashes.size() * 10) { - capacity *= 2; - } - key_index_.rehash(capacity); - packed_fast_path_ = false; - } - - // ── Multi-key categorical index, keyed on the code tuple ────────────────── - // - // Groups are identified by the codes themselves (stored in - // multi_cat_codes_flat_), verified on every hit. The Cartesian cell is only - // a usable identity while the stride product fits in 64 bits; this does not - // care, and it survives dictionary growth without a rebuild. - static auto hash_codes(const Column::code_type* codes, std::size_t n) - -> std::uint64_t { - std::uint64_t seed = 0; - for (std::size_t i = 0; i < n; ++i) { - key_hash_mix(seed, std::hash{}(static_cast(codes[i]))); - } - // Finalized for the same reason `hash_key_row` is: this index masks the - // LOW bits to pick a slot and probes linearly, and the combine above - // never diffuses into them. Categorical codes are small dense integers, - // which is precisely the input that makes the unfinalized combine a - // near-linear function of the key and turns the probe into one long - // cluster. Nothing outside this index consumes the value, so unlike the - // three in `interpreter_internal.hpp` it has no agreement to maintain. - return key_hash_finalize(seed); - } - - [[nodiscard]] auto codes_of_group(std::size_t group, std::size_t n_keys) const - -> const Column::code_type* { - return multi_cat_codes_flat_.data() + (group * n_keys); - } - - void multi_cat_rehash_slots(std::size_t capacity, std::size_t n_keys) { - multi_cat_slots_.assign(capacity, 0U); - const std::size_t mask = capacity - 1; - for (std::size_t group = 0; group < n_groups_; ++group) { - std::size_t probe = - static_cast(hash_codes(codes_of_group(group, n_keys), n_keys)) & mask; - while (multi_cat_slots_[probe] != 0) { - probe = (probe + 1) & mask; - } - multi_cat_slots_[probe] = static_cast(group) + 1; - } - } - - /// Rebuild the index from the groups already collected — used when the - /// dense array gives up, and to seed the table on first use. - void multi_cat_rehash_groups() { - const std::size_t n_keys = n_groups_ == 0 ? 0 : multi_cat_codes_flat_.size() / n_groups_; - std::size_t capacity = 1024; - while ((n_groups_ * 10) > (capacity * 7)) { - capacity *= 2; - } - multi_cat_rehash_slots(capacity, n_keys); - } - - template - auto multi_cat_find_or_insert(const Column::code_type* codes, std::size_t n_keys, - NewGroup&& new_group) -> std::uint32_t { - const std::uint64_t hash = hash_codes(codes, n_keys); - std::size_t mask = multi_cat_slots_.size() - 1; - std::size_t probe = static_cast(hash) & mask; - while (true) { - const std::uint32_t slot = multi_cat_slots_[probe]; - if (slot == 0) { - const std::uint32_t gid = new_group(); - multi_cat_slots_[probe] = gid + 1; - if ((n_groups_ * 10) > (multi_cat_slots_.size() * 7)) { - multi_cat_rehash_slots(multi_cat_slots_.size() * 2, n_keys); - } - return gid; - } - const std::uint32_t gid = slot - 1; - if (std::equal(codes, codes + n_keys, codes_of_group(gid, n_keys))) { - return gid; - } - probe = (probe + 1) & mask; - } - } - - auto process_rows_cat(const std::vector& group_entries, - const std::vector& agg_entries, std::size_t rows) - -> std::optional { - std::vector*> cat_cols; - cat_cols.reserve(group_entries.size()); - for (const auto* e : group_entries) { - cat_cols.push_back(&std::get>(*e->column)); - } - const std::size_t n_keys = cat_cols.size(); - const bool single_key = n_keys == 1; - - if (single_key && rows > 0 && - try_process_rows_cat_parallel(*cat_cols[0], agg_entries, rows)) { - return std::nullopt; - } - - gids_buf_.resize(rows); - auto* gids = gids_buf_.data(); - if (single_key) { - // A Categorical code is already a dense index into [0, dict_size), - // so map code → gid with a direct array instead of hashing. Dicts - // only grow and never reorder across chunks, so existing gids stay - // valid and new dict entries just extend the array with sentinels. - const auto* codes = cat_cols[0]->codes_data(); - const std::size_t dict_size = cat_cols[0]->dictionary().size(); - if (cat_dense_gid_.size() < dict_size) { - cat_dense_gid_.resize(dict_size, kNoGid); - } - std::uint32_t* dense = cat_dense_gid_.data(); - for (std::size_t row = 0; row < rows; ++row) { - const auto code = codes[row]; - std::uint32_t gid = dense[code]; - if (gid == kNoGid) { - gid = alloc_group(); - dense[code] = gid; - cat_order_.push_back(code); - } - gids[row] = gid; - } - } else { - // Multi-key: encode each row as a uint64_t Cartesian cell. - // Strides may grow across chunks if a chunk introduces new dict - // entries; we recompute per chunk and rebuild the index when that - // happens (rare — Categorical dicts are usually stable). - std::vector dict_sizes(n_keys); - for (std::size_t c = 0; c < n_keys; ++c) { - dict_sizes[c] = static_cast(cat_cols[c]->dictionary().size()); - if (dict_sizes[c] == 0) - dict_sizes[c] = 1; // avoid stride collapse - } - // Strides: cell = c0*s0 + c1*s1 + … with s_{n-1} = 1. - // - // The cell only identifies a key tuple while the stride product - // fits in 64 bits. Past that the multiply wraps, distinct tuples - // collide, and `total_cells` itself wraps — a product of exactly - // 2^64 (16 keys of 16 values, say) lands on 0, which would pass the - // dense-array bound and index a zero-length array. Detect the - // overflow and let the hash path, which identifies groups by their - // codes rather than by a cell, take over. - std::vector strides(n_keys); - std::uint64_t total_cells = 1; - bool cells_overflow = false; - { - std::uint64_t s = 1; - for (int ci = static_cast(n_keys) - 1; ci >= 0; --ci) { - strides[static_cast(ci)] = s; - const std::uint64_t size = dict_sizes[static_cast(ci)]; - if (s > std::numeric_limits::max() / size) { - cells_overflow = true; - break; - } - s *= size; - } - total_cells = s; - } - const bool dense_possible = !cells_overflow && total_cells <= kDenseCellLimit; - - // Hoist raw code pointers out of the row loop. - std::vector::code_type*> raws(n_keys); - for (std::size_t c = 0; c < n_keys; ++c) - raws[c] = cat_cols[c]->codes_data(); - - const auto cell_of_group = [&](std::size_t g) -> std::uint64_t { - std::uint64_t cell = 0; - for (std::size_t c = 0; c < n_keys; ++c) { - cell += static_cast(multi_cat_codes_flat_[(g * n_keys) + c]) * - strides[c]; - } - return cell; - }; - const auto new_group = [&](std::size_t row) -> std::uint32_t { - for (std::size_t c = 0; c < n_keys; ++c) - multi_cat_codes_flat_.push_back(raws[c][row]); - return alloc_group(); - }; - - // When the Cartesian cell space is bounded, index a dense array - // (one load per row, no hashing). If a later chunk grows the dicts - // past the limit — or past what 64 bits can encode — migrate the - // existing groups into the hash index once and stay there; dicts - // only grow, so the cell space never shrinks back. - if (multi_dense_ && !dense_possible) { - multi_cat_rehash_groups(); - std::vector().swap(multi_cat_cell_dense_); - multi_dense_ = false; - } - - if (multi_dense_) { - // Rebuild the dense array when strides change (new dict entries). - if (multi_cat_strides_ != strides) { - multi_cat_cell_dense_.assign(static_cast(total_cells), kNoGid); - for (std::size_t g = 0; g < n_groups_; ++g) - multi_cat_cell_dense_[cell_of_group(g)] = static_cast(g); - multi_cat_strides_ = strides; - } - std::uint32_t* dense = multi_cat_cell_dense_.data(); - if (n_keys == 2) { - const auto* k0 = raws[0]; - const auto* k1 = raws[1]; - const std::uint64_t s0 = strides[0]; - const std::uint64_t s1 = strides[1]; - for (std::size_t row = 0; row < rows; ++row) { - const std::uint64_t cell = (static_cast(k0[row]) * s0) + - (static_cast(k1[row]) * s1); - std::uint32_t gid = dense[cell]; - if (gid == kNoGid) { - gid = new_group(row); - dense[cell] = gid; - } - gids[row] = gid; - } - } else { - for (std::size_t row = 0; row < rows; ++row) { - std::uint64_t cell = 0; - for (std::size_t c = 0; c < n_keys; ++c) - cell += static_cast(raws[c][row]) * strides[c]; - std::uint32_t gid = dense[cell]; - if (gid == kNoGid) { - gid = new_group(row); - dense[cell] = gid; - } - gids[row] = gid; - } - } - } else { - // Hash fallback for cell spaces that are unbounded, or that no - // longer fit in 64 bits. It identifies a group by its codes, not - // by a cell: correct however the strides behave, and it needs no - // rebuild when a new dictionary entry changes them. - if (multi_cat_slots_.empty()) { - multi_cat_rehash_groups(); - } - std::vector::code_type> row_codes(n_keys); - for (std::size_t row = 0; row < rows; ++row) { - for (std::size_t c = 0; c < n_keys; ++c) { - row_codes[c] = raws[c][row]; - } - gids[row] = multi_cat_find_or_insert(row_codes.data(), n_keys, - [&] { return new_group(row); }); - } - } - } - - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - /// Row count below which partitioned discovery is not worth its fan-out, - /// for the int/string keys it was originally tuned against. - static constexpr std::size_t kDefaultPartitionMinRows = 1U << 18U; - /// The packed key's break-even is lower: its serial probe hashes and - /// compares a multi-column key, so there is more per-row work to move onto - /// the workers than a single int key offers. - static constexpr std::size_t kPackedPartitionMinRows = 1U << 15U; - - /// Everything one packed width needs on the group-by side. - template - struct PackedGroups { - robin_hood::unordered_flat_map index; - std::vector> partitions; - }; - - /// Three or more fixed-width key columns, grouped through one packed key. - /// - /// **The output key store is deliberately unchanged.** `group_order_` still - /// holds one boxed `Key` per group, built once per GROUP, so - /// `build_output_chunk` needs no packed case: this path sets none of the - /// `*_fast_path_` flags and lands in the same branch the generic path uses. - /// That is also why the key is rebuilt from the ROW rather than unpacked — - /// a packed key is not invertible on its own, since a Categorical cell - /// holds an operator-global interned id rather than the column's own code. - template - auto process_rows_packed(const std::vector& group_entries, - const std::vector& agg_entries, - const std::vector& cols, std::size_t rows, - PackedGroups& state) -> std::optional { - gids_buf_.resize(rows); - auto* gids = gids_buf_.data(); - - // The one place a Key gets built: once per group, never per row. - const auto build_key_at = [&](std::size_t row) { - Key key; - key.values.reserve(group_entries.size()); - for (const auto* entry : group_entries) { - push_key_value(key, *entry, row); - } - return key; - }; - const auto key_at = [&](std::size_t row) { - return PackedKeyEncoder::pack_row(cols, row); - }; - - if (try_discover_partitioned( - key_at, rows, gids, state.partitions, - [&](std::size_t n) { group_order_.resize(n); }, - [&](const Packed&, std::uint32_t gid, std::size_t row) { - group_order_[gid] = build_key_at(row); - }, - kPackedPartitionMinRows)) { - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - // Run-length shortcut, as in the string and int paths: sorted or chunked - // input often repeats the key, so skip the map lookup when it matches - // the previous row. - Packed prev_key{}; - std::uint32_t prev_gid = std::numeric_limits::max(); - bool have_prev = false; - for (std::size_t row = 0; row < rows; ++row) { - const Packed key = key_at(row); - std::uint32_t gid{}; - if (have_prev && key == prev_key) { - gid = prev_gid; - } else { - auto it = state.index.find(key); - if (it == state.index.end()) { - group_order_.push_back(build_key_at(row)); - gid = alloc_group(); - state.index.emplace(key, gid); - } else { - gid = it->second; - } - prev_key = key; - prev_gid = gid; - have_prev = true; - } - gids[row] = gid; - } - - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - auto process_rows_generic(const std::vector& group_entries, - const std::vector& agg_entries, std::size_t rows) - -> std::optional { - std::vector cols; - cols.reserve(group_entries.size()); - for (const auto* entry : group_entries) { - auto col = make_key_col(*entry); - if (!col.has_value()) { - return "group-by: unsupported key column type"; - } - cols.push_back(*col); - } - - gids_buf_.resize(rows); - auto* gids = gids_buf_.data(); - for (std::size_t row = 0; row < rows; ++row) { - gids[row] = key_index_.find_or_insert(group_order_, cols, row, [&] { - // The one place a Key gets built: once per group, not per row. - Key key; - key.values.reserve(group_entries.size()); - for (const auto* entry : group_entries) { - push_key_value(key, *entry, row); - } - group_order_.push_back(std::move(key)); - return alloc_group(); - }); - } - - accumulate_gids(gids, agg_entries, rows); - return std::nullopt; - } - - /// Store non-null First aggregates at the row that creates each new group. - /// `seeded[a]` is set only when the input column has no validity bitmap, - /// so the discovery row is necessarily the first value under First's - /// null-skipping semantics. The later accumulation pass may skip exactly - /// these fields; nullable First and every Last still scan normally. - void seed_discovery_first(std::size_t first_gid, const std::vector& first_rows, - const std::vector& agg_entries, - const std::vector& seeded) { - bool has_text = false; - for (std::size_t a = 0; a < n_aggs_; ++a) { - has_text = has_text || (seeded[a] != 0U && plan_[a].kind == ExprType::String); - } - if (has_text && text_store_.size() < flat_slots_.size()) { - text_store_.resize(flat_slots_.size()); - } - const auto seed_one = [&](std::size_t gid, std::size_t row, std::size_t a) { - const auto& entry = *agg_entries[a]; - auto& slot = flat_slots_[(gid * n_aggs_) + a]; - if (plan_[a].kind == ExprType::Double) { - slot.double_value = std::get>(*entry.column)[row]; - } else if (plan_[a].kind == ExprType::Int) { - slot.int_value = std::get>(*entry.column)[row]; - } else { - std::string value; - if (plan_[a].categorical) { - value = std::string(std::get>(*entry.column)[row]); - } else { - value = std::string(std::get>(*entry.column)[row]); - } - text_store_[(gid * n_aggs_) + a] = std::move(value); - } - slot.mark_present(); - }; - - // Ceiling and permission from the plan; the `parallel_min_rows` floor on - // the first-occurrence count stays here (the shared knob, not a phase - // constant). - std::size_t threads = 1; - if (exec_ != nullptr && par_.finalize.decline == physical::FanOutDecline::None && - !on_worker_pool_thread() && first_rows.size() >= exec_->parallel_min_rows) { - threads = par_.finalize.worker_cap; - } - if (threads >= 2) { - note_finalize_fanout(); - const std::size_t grain = (first_rows.size() + threads - 1) / threads; - auto batch = process_worker_pool().submit(threads, [&](std::size_t worker) { - const std::size_t begin = worker * grain; - const std::size_t end = std::min(first_rows.size(), begin + grain); - for (std::size_t local = begin; local < end; ++local) { - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (seeded[a] != 0U) { - seed_one(first_gid + local, first_rows[local], a); - } - } - } - }); - batch.wait(); - if (exec_->parallel_stats != nullptr) { - exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); - } - return; - } - - // Field-major serial order keeps each source column's reads together. - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (seeded[a] == 0U) { - continue; - } - for (std::size_t local = 0; local < first_rows.size(); ++local) { - seed_one(first_gid + local, first_rows[local], a); - } - } - } - - /// Parallel scatter-accumulate of an already-assigned gid array — the - /// shared back half of every hash group-by fast path (string, int, - /// int-pair, generic). Returns false when the shape is not worth it and - /// the caller should accumulate serially into `flat_slots_`. - /// - /// **The gid pass stays serial on purpose.** It mutates the group index - /// and it is what defines group ORDER — Ibex reports groups in observed - /// first-occurrence order, so assigning gids concurrently would either - /// change the answer or need a reconciliation pass costing more than the - /// scan. Once every row carries a gid the rest is a pure scatter-reduce, - /// which is the part worth threading: for an 8-aggregate query like q01 it - /// is the dominant cost (24% of the whole query by profile), while the gid - /// probe is a single packed-integer lookup per row. - /// - /// Reproducibility, stated exactly, because the two halves get confused: - /// the partition is derived from row count and group count alone — both - /// properties of the DATA — and morsels merge in ascending order, so the - /// result does not depend on the machine, the thread count, or the - /// schedule (verified byte-identical across 2/3/5/8/16 threads). It DOES - /// differ from the serial path in the last ulp, because summing per morsel - /// and merging is not the same order as summing down the rows — q01's - /// sum_disc_price moves at the 11th significant digit. That is inherent to - /// any partitioned float reduction, it matches what - /// `try_process_rows_cat_parallel` has always done, and if anything the - /// partitioned sum is the more accurate of the two. - /// - /// This mirrors `try_process_rows_cat_parallel`, which can skip the gid - /// pass entirely because a Categorical code is already a dense index. - auto try_accumulate_parallel(const std::uint32_t* gids, - const std::vector& agg_entries, - std::size_t rows, const std::vector* skip = nullptr) - -> bool { - // Partition on the data alone -- not `exec_->can_fan_out()`, the thread - // budget, or whether this runs on a pool thread. Those choose who - // executes the morsels; the cut decides the arithmetic, and a cut that - // varied with the schedule would let the same query answer differently - // on two machines. Morsels run inline below when fan-out is not - // available. - if (n_groups_ == 0 || n_aggs_ == 0) { - return false; - } - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (skip != nullptr && (*skip)[a] != 0U) { - continue; - } - if (!agg_is_combinable(plan_[a].func)) { - return false; - } - // A boxed First/Last value lives outside the slot array, so a - // private copy would not capture it. - if (plan_[a].kind != ExprType::Int && plan_[a].kind != ExprType::Double) { - return false; - } - } - - // Same budget and morsel shape as the Categorical path, for the same - // reasons: partial state is bounded by GROUP COUNT, and the merge costs - // one agg_combine per (morsel, group) while the scan it replaces costs - // one update per row. Fanning out only pays when the merge stays small - // against the scan — a high-cardinality group-by would merge more slots - // than it saved row updates. - constexpr std::size_t kMinRowsPerMorsel = 65536; - constexpr std::size_t kMaxMorsels = 64; - constexpr std::size_t kPartialBudgetBytes = 32UL << 20; - constexpr std::size_t kMergeToScanRatio = 4; - const std::size_t per_morsel_bytes = - n_groups_ * ((n_aggs_ * sizeof(AggSlotCore)) + (scratch_stride_ * sizeof(double))); - if (per_morsel_bytes == 0 || per_morsel_bytes > kPartialBudgetBytes) { - return false; - } - std::size_t morsels = std::clamp(rows / kMinRowsPerMorsel, 1, kMaxMorsels); - morsels = std::min(morsels, kPartialBudgetBytes / per_morsel_bytes); - if (morsels < 2 || morsels * n_groups_ > rows / kMergeToScanRatio) { - return false; - } - - const std::size_t stride = n_groups_ * n_aggs_; - const std::size_t scratch_span = n_groups_ * scratch_stride_; - const std::size_t grain = (rows + morsels - 1) / morsels; - std::vector partials(morsels * stride); - std::vector partial_scratch(morsels * scratch_span, 0.0); - - const auto run_morsel = [&](std::size_t m) { - const std::size_t begin = m * grain; - const std::size_t end = std::min(rows, begin + grain); - if (begin < end) { - accumulate_columns_into(gids, agg_entries, begin, end, &partials[m * stride], - partial_scratch.data() + (m * scratch_span), skip); - } - }; - const std::size_t threads = - std::min(morsels, exec_ != nullptr ? exec_->compute_budget() : std::size_t{1}); - const bool fanned_out = threads >= 2 && !on_worker_pool_thread(); - if (fanned_out) { - auto& pool = process_worker_pool(); - std::atomic cursor{0}; - auto batch = pool.submit(threads, [&](std::size_t) { - while (true) { - const std::size_t m = cursor.fetch_add(1, std::memory_order_relaxed); - if (m >= morsels) { - return; - } - run_morsel(m); - } - }); - batch.wait(); - } else { - for (std::size_t m = 0; m < morsels; ++m) { - run_morsel(m); - } - } - - for (std::size_t m = 0; m < morsels; ++m) { - const AggSlotCore* src = &partials[m * stride]; - const double* src_scratch = partial_scratch.data() + (m * scratch_span); - for (std::size_t g = 0; g < n_groups_; ++g) { - AggSlotCore* dst = &flat_slots_[g * n_aggs_]; - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (skip != nullptr && (*skip)[a] != 0U) { - continue; - } - const std::size_t off = (g * scratch_stride_) + scratch_offset_[a]; - agg_combine(dst[a], src[(g * n_aggs_) + a], plan_[a].func, plan_[a].kind, - scratch_stride_ == 0 ? nullptr : scratch_.data() + off, - scratch_stride_ == 0 ? nullptr : src_scratch + off); - } - } - } - if (fanned_out && exec_ != nullptr && exec_->parallel_stats != nullptr) { - // Counts a fan-out, not a partition. The morsels are cut the same - // way either way, so counting them when they ran inline would - // report parallelism that never happened. - exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); - } - return true; - } - - /// Accumulate `gids` either across workers or, when that is not worth it, - /// serially — the one call every gid-assigning fast path ends with. - void accumulate_gids(const std::uint32_t* gids, - const std::vector& agg_entries, std::size_t rows, - const std::vector* skip = nullptr) { - if (!try_accumulate_parallel(gids, agg_entries, rows, skip)) { - accumulate_columns_into(gids, agg_entries, 0, rows, flat_slots_.data(), scratch_.data(), - skip); - } - } - - /// Scatter-accumulate rows [begin, end) into `base`, indexed by - /// `gids[row] * n_aggs_ + agg_i`. `base` is the caller's slot array — - /// `flat_slots_` for the serial path, a worker-private array for the - /// parallel one — and `GidT` covers both assigned gids (uint32_t) and raw - /// Categorical codes (int32_t), which are already dense indices. - /// - template - void accumulate_columns_into(const GidT* gids, - const std::vector& agg_entries, - std::size_t begin, std::size_t end, AggSlotCore* base, - double* scratch_base, - const std::vector* skip = nullptr) { - AggSlotCore* fs = base; - const std::size_t rows = end; - for (std::size_t agg_i = 0; agg_i < n_aggs_; ++agg_i) { - if (skip != nullptr && (*skip)[agg_i] != 0U) { - continue; - } - // Takes GidT so a signed Categorical code indexes without an - // implicit narrowing conversion at each of the ~19 call sites. - const auto slot_for = [&](GidT g) -> AggSlotCore& { - return fs[(static_cast(g) * n_aggs_) + agg_i]; - }; - // Moment accumulators, laid out beside `base` and indexed the same - // way. A worker-private slot array needs a worker-private scratch - // to match, or two morsels would accumulate one group's variance - // into the same doubles. - const auto scratch_at = [&](GidT g) -> double* { - return scratch_base + (static_cast(g) * scratch_stride_) + - scratch_offset_[agg_i]; - }; - - if (plan_[agg_i].func == ir::AggFunc::Count) { - for (std::size_t row = begin; row < rows; ++row) { - slot_for(gids[row]).count++; - } - continue; - } - - const auto& entry = *agg_entries[agg_i]; - const ValidityBitmap* validity = - entry.validity.has_value() ? &*entry.validity : nullptr; - const bool has_nulls = validity != nullptr; - - if (plan_[agg_i].kind == ExprType::Double) { - const double* data = std::get>(*entry.column).data(); - switch (plan_[agg_i].func) { - case ir::AggFunc::Sum: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - slot.double_value += data[row]; - slot.mark_present(); - } - break; - case ir::AggFunc::Mean: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - slot.double_value += data[row]; - slot.count++; - } - break; - case ir::AggFunc::Min: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - const double v = data[row]; - slot.double_value = slot.present() ? std::min(slot.double_value, v) : v; - slot.mark_present(); - } - break; - case ir::AggFunc::Max: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - const double v = data[row]; - slot.double_value = slot.present() ? std::max(slot.double_value, v) : v; - slot.mark_present(); - } - break; - case ir::AggFunc::Stddev: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - agg_update_stddev(slot_for(gids[row]), scratch_at(gids[row])[0], - data[row]); - } - break; - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - double* scr = scratch_at(gids[row]); - agg_update_moments(slot_for(gids[row]), scr[0], scr[1], scr[2], - data[row]); - } - break; - case ir::AggFunc::First: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - if (!slot.present()) { - slot.double_value = data[row]; - slot.mark_present(); - } - } - break; - case ir::AggFunc::Last: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - slot.double_value = data[row]; - slot.mark_present(); - } - break; - default: - break; - } - } else if (plan_[agg_i].kind == ExprType::Int) { - const std::int64_t* data = std::get>(*entry.column).data(); - switch (plan_[agg_i].func) { - case ir::AggFunc::Sum: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - slot.int_value += data[row]; - slot.mark_present(); - } - break; - case ir::AggFunc::Mean: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - slot.double_value += static_cast(data[row]); - slot.count++; - } - break; - case ir::AggFunc::Min: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - std::int64_t v = data[row]; - slot.int_value = slot.present() ? std::min(slot.int_value, v) : v; - slot.mark_present(); - } - break; - case ir::AggFunc::Max: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - std::int64_t v = data[row]; - slot.int_value = slot.present() ? std::max(slot.int_value, v) : v; - slot.mark_present(); - } - break; - case ir::AggFunc::Stddev: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - agg_update_stddev(slot_for(gids[row]), scratch_at(gids[row])[0], - static_cast(data[row])); - } - break; - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - double* scr = scratch_at(gids[row]); - agg_update_moments(slot_for(gids[row]), scr[0], scr[1], scr[2], - static_cast(data[row])); - } - break; - case ir::AggFunc::First: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - if (!slot.present()) { - slot.int_value = data[row]; - slot.mark_present(); - } - } - break; - case ir::AggFunc::Last: - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - slot.int_value = data[row]; - slot.mark_present(); - } - break; - default: - break; - } - } else { - // ExprType::String — First/Last only (the type gate in - // process_chunk rejects every other function here). Covers - // both Column and Column; the two - // share ScalarValue{std::string} as the wire format via - // append_scalar, which pushes into a Categorical dictionary - // when the target column is Categorical. - const bool categorical = plan_[agg_i].categorical; - const auto value_at = [&](std::size_t row) -> std::string { - if (categorical) { - return std::string(std::get>(*entry.column)[row]); - } - return std::string(std::get>(*entry.column)[row]); - }; - if (plan_[agg_i].func == ir::AggFunc::First) { - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - if (!slot.present()) { - text_at((static_cast(gids[row]) * n_aggs_) + agg_i) = - value_at(row); - slot.mark_present(); - } - } - } else { - for (std::size_t row = begin; row < rows; ++row) { - if (has_nulls && !(*validity)[row]) - continue; - auto& slot = slot_for(gids[row]); - text_at((static_cast(gids[row]) * n_aggs_) + agg_i) = - value_at(row); - slot.mark_present(); - } - } - } - } - } - - /// Accumulate rows [begin, end) of a global aggregate into `slots` - /// (n_aggs_ entries, caller-owned). No gid indirection: the compiler sees - /// a plain reduction over a contiguous range, which is also what lets a - /// worker own a private copy. - /// Parallel single-key Categorical group-by. Returns false when the shape - /// is not eligible and the caller should run the serial path. - /// - /// A Categorical code is already a dense index into the dictionary, so a - /// worker needs no hash table at all: it accumulates into a private slot - /// array indexed by code. That bounds the partial state by DICTIONARY - /// SIZE, which is what makes per-worker group state affordable here and - /// keeps `by symbol` (a few hundred groups) cheap. - /// - /// Group order is Ibex's observed first-occurrence order, and the merge - /// preserves it exactly: morsels are contiguous ascending row ranges, and - /// merging them in ascending order while walking each morsel's own - /// first-seen code list visits codes in precisely the order a serial scan - /// would have met them. - auto try_process_rows_cat_parallel(const Column& cat, - const std::vector& agg_entries, - std::size_t rows) -> bool { - // Partition on the data alone -- not `exec_->can_fan_out()`, the thread - // budget, or whether this runs on a pool thread. Those choose who - // executes the morsels; the cut decides the arithmetic, and a cut that - // varied with the schedule would let the same query answer differently - // on two machines. Morsels run inline below when fan-out is not - // available. - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (!agg_is_combinable(plan_[a].func)) { - return false; - } - // As above: a boxed First/Last value lives outside the slot. - if (plan_[a].kind != ExprType::Int && plan_[a].kind != ExprType::Double) { - return false; - } - } - const std::size_t dict_size = cat.dictionary().size(); - if (dict_size == 0) { - return false; - } - - // Same row-derived partition as the global aggregate, then bounded by - // what the per-worker slot arrays cost. Both inputs (row count and - // dictionary size) are properties of the DATA, so the partition — and - // therefore the float reduction order — is still independent of the - // machine and the schedule. - constexpr std::size_t kMinRowsPerMorsel = 65536; - constexpr std::size_t kMaxMorsels = 64; - constexpr std::size_t kPartialBudgetBytes = 32UL << 20; - const std::size_t per_morsel_bytes = - dict_size * ((n_aggs_ * sizeof(AggSlotCore)) + (scratch_stride_ * sizeof(double))); - if (per_morsel_bytes == 0 || per_morsel_bytes > kPartialBudgetBytes) { - return false; // one worker's state alone blows the budget - } - std::size_t morsels = std::clamp(rows / kMinRowsPerMorsel, 1, kMaxMorsels); - morsels = std::min(morsels, kPartialBudgetBytes / per_morsel_bytes); - if (morsels < 2) { - return false; - } - // The merge costs one agg_combine per (morsel, dictionary entry), so it - // scales with GROUP COUNT while the scan it replaces scales with rows. - // Fanning out only pays when the merge stays small against the scan: - // `by symbol` (252 groups) merges ~4k slots against 1M rows, but - // `by user_id` (100k groups) would merge ~1M — more work than it saves, - // and measured as a 17% REGRESSION when a smaller slot let it through - // the memory gate. - constexpr std::size_t kMergeToScanRatio = 4; - if (morsels * dict_size > rows / kMergeToScanRatio) { - return false; - } - - const auto* codes = cat.codes_data(); - const std::size_t grain = (rows + morsels - 1) / morsels; - std::vector partials(morsels * dict_size * n_aggs_); - std::vector cat_partial_scratch(morsels * dict_size * scratch_stride_, 0.0); - // Per morsel, the codes it saw in first-occurrence order. - std::vector::code_type>> seen(morsels); - - const auto run_morsel = [&](std::size_t m, std::vector& local_seen) { - const std::size_t begin = m * grain; - const std::size_t end = std::min(rows, begin + grain); - if (begin >= end) { - return; - } - std::ranges::fill(local_seen, std::uint8_t{0}); - auto& order = seen[m]; - for (std::size_t row = begin; row < end; ++row) { - const auto code = codes[row]; - if (local_seen[static_cast(code)] == 0) { - local_seen[static_cast(code)] = 1; - order.push_back(code); - } - } - accumulate_columns_into(codes, agg_entries, begin, end, - &partials[m * dict_size * n_aggs_], - cat_partial_scratch.data() + (m * dict_size * scratch_stride_)); - }; - const std::size_t threads = - std::min(morsels, exec_ != nullptr ? exec_->compute_budget() : std::size_t{1}); - const bool fanned_out = threads >= 2 && !on_worker_pool_thread(); - if (fanned_out) { - auto& pool = process_worker_pool(); - std::atomic cursor{0}; - auto batch = pool.submit(threads, [&](std::size_t) { - std::vector local_seen(dict_size, 0); - while (true) { - const std::size_t m = cursor.fetch_add(1, std::memory_order_relaxed); - if (m >= morsels) { - return; - } - run_morsel(m, local_seen); - } - }); - batch.wait(); - } else { - std::vector local_seen(dict_size, 0); - for (std::size_t m = 0; m < morsels; ++m) { - run_morsel(m, local_seen); - } - } - - if (cat_dense_gid_.size() < dict_size) { - cat_dense_gid_.resize(dict_size, kNoGid); - } - for (std::size_t m = 0; m < morsels; ++m) { - const AggSlotCore* src = &partials[m * dict_size * n_aggs_]; - const double* src_scratch = - cat_partial_scratch.data() + (m * dict_size * scratch_stride_); - for (const auto code : seen[m]) { - const auto idx = static_cast(code); - std::uint32_t gid = cat_dense_gid_[idx]; - if (gid == kNoGid) { - gid = alloc_group(); - cat_dense_gid_[idx] = gid; - cat_order_.push_back(code); - } - AggSlotCore* dst = &flat_slots_[(static_cast(gid) * n_aggs_)]; - for (std::size_t a = 0; a < n_aggs_; ++a) { - agg_combine(dst[a], src[(idx * n_aggs_) + a], plan_[a].func, plan_[a].kind, - scratch_stride_ == 0 ? nullptr : scratch_for(gid, a), - scratch_stride_ == 0 - ? nullptr - : src_scratch + (idx * scratch_stride_) + scratch_offset_[a]); - } - } - } - if (fanned_out && exec_ != nullptr && exec_->parallel_stats != nullptr) { - // Counts a fan-out, not a partition. The morsels are cut the same - // way either way, so counting them when they ran inline would - // report parallelism that never happened. - exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); - } - return true; - } - - /// `scratch_base` is this caller's moment region for the single group — - /// `scratch_` serially, a worker-private slice per morsel in parallel. - void accumulate_ungrouped_range_impl(const std::vector& agg_entries, - std::size_t begin, std::size_t end, AggSlotCore* slots, - double* scratch_base) { - for (std::size_t agg_i = 0; agg_i < n_aggs_; ++agg_i) { - AggSlotCore& slot = slots[agg_i]; - const auto func = plan_[agg_i].func; - if (func == ir::AggFunc::Count) { - slot.count += static_cast(end - begin); - continue; - } - const auto& entry = *agg_entries[agg_i]; - const ValidityBitmap* validity = - entry.validity.has_value() ? &*entry.validity : nullptr; - const bool has_nulls = validity != nullptr; - - // One generic driver per storage kind; `step` is the per-row body. - const auto each = [&](auto&& step) { - if (has_nulls) { - for (std::size_t row = begin; row < end; ++row) { - if (!(*validity)[row]) { - continue; - } - step(row); - } - } else { - for (std::size_t row = begin; row < end; ++row) { - step(row); - } - } - }; - - if (plan_[agg_i].kind == ExprType::Double) { - const double* data = std::get>(*entry.column).data(); - switch (func) { - case ir::AggFunc::Sum: - // has_value must track "saw a non-null value", not - // "the range was non-empty": sum over an all-null - // column is NULL, not 0. - each([&](std::size_t r) { - slot.double_value += data[r]; - slot.mark_present(); - }); - break; - case ir::AggFunc::Mean: - each([&](std::size_t r) { - slot.double_value += data[r]; - slot.count++; - }); - break; - case ir::AggFunc::Min: - each([&](std::size_t r) { - slot.double_value = - slot.present() ? std::min(slot.double_value, data[r]) : data[r]; - slot.mark_present(); - }); - break; - case ir::AggFunc::Max: - each([&](std::size_t r) { - slot.double_value = - slot.present() ? std::max(slot.double_value, data[r]) : data[r]; - slot.mark_present(); - }); - break; - case ir::AggFunc::Stddev: - each([&](std::size_t r) { - agg_update_stddev(slot, scratch_base[scratch_offset_[agg_i]], data[r]); - }); - break; - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - each([&](std::size_t r) { - double* scr = scratch_base + scratch_offset_[agg_i]; - agg_update_moments(slot, scr[0], scr[1], scr[2], data[r]); - }); - break; - case ir::AggFunc::First: - each([&](std::size_t r) { - if (!slot.present()) { - slot.double_value = data[r]; - slot.mark_present(); - } - }); - break; - case ir::AggFunc::Last: - each([&](std::size_t r) { - slot.double_value = data[r]; - slot.mark_present(); - }); - break; - default: - break; - } - } else if (plan_[agg_i].kind == ExprType::Int) { - const std::int64_t* data = std::get>(*entry.column).data(); - switch (func) { - case ir::AggFunc::Sum: - each([&](std::size_t r) { - slot.int_value += data[r]; - slot.mark_present(); - }); - break; - case ir::AggFunc::Mean: - each([&](std::size_t r) { - slot.double_value += static_cast(data[r]); - slot.count++; - }); - break; - case ir::AggFunc::Min: - each([&](std::size_t r) { - slot.int_value = - slot.present() ? std::min(slot.int_value, data[r]) : data[r]; - slot.mark_present(); - }); - break; - case ir::AggFunc::Max: - each([&](std::size_t r) { - slot.int_value = - slot.present() ? std::max(slot.int_value, data[r]) : data[r]; - slot.mark_present(); - }); - break; - case ir::AggFunc::Stddev: - each([&](std::size_t r) { - agg_update_stddev(slot, scratch_base[scratch_offset_[agg_i]], - static_cast(data[r])); - }); - break; - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - each([&](std::size_t r) { - double* scr = scratch_base + scratch_offset_[agg_i]; - agg_update_moments(slot, scr[0], scr[1], scr[2], - static_cast(data[r])); - }); - break; - case ir::AggFunc::First: - each([&](std::size_t r) { - if (!slot.present()) { - slot.int_value = data[r]; - slot.mark_present(); - } - }); - break; - case ir::AggFunc::Last: - each([&](std::size_t r) { - slot.int_value = data[r]; - slot.mark_present(); - }); - break; - default: - break; - } - } else { - // ExprType::String — First/Last only, same wire format as the - // grouped path (ScalarValue{std::string}). - const bool categorical = plan_[agg_i].categorical; - const auto value_at = [&](std::size_t row) -> std::string { - if (categorical) { - return std::string(std::get>(*entry.column)[row]); - } - return std::string(std::get>(*entry.column)[row]); - }; - if (func == ir::AggFunc::First) { - each([&](std::size_t r) { - if (!slot.present()) { - text_at(agg_i) = value_at(r); - slot.mark_present(); - } - }); - } else { - each([&](std::size_t r) { - text_at(agg_i) = value_at(r); - slot.mark_present(); - }); - } - } - } - } - - /// Global aggregate over `rows`, optionally fanned out across workers. - auto process_rows_ungrouped(const std::vector& agg_entries, - std::size_t rows) -> std::optional { - // An empty input must produce NO group, hence no output row — the - // generic path got that for free by only creating a group when a row - // arrived. Creating it up front turned `count()` over an empty table - // into a 1-row answer. - if (rows == 0) { - return std::nullopt; - } - if (n_groups_ == 0) { - // build_output_chunk() reads group_order_[g] for the generic key - // layout, so the single group still needs its (empty) Key — the - // generic path used to push one from its make_group lambda. - group_order_.emplace_back(); - alloc_group(); - } - AggSlotCore* dst = flat_slots_.data(); - - const std::size_t morsels = ungrouped_morsels(rows); - if (morsels < 2) { - accumulate_ungrouped_range_impl(agg_entries, 0, rows, dst, scratch_.data()); - return std::nullopt; - } - - // One private slot array per morsel, each written by exactly one - // worker. Merging them in ascending morsel order — never completion - // order — is what keeps First/Last correct and the float reduction - // reproducible run to run. - const std::size_t grain = (rows + morsels - 1) / morsels; - std::vector partials(morsels * n_aggs_); - std::vector ung_scratch(morsels * scratch_stride_, 0.0); - - // One morsel's work, identical whoever runs it -- which is the point: - // the partial it writes and the slot it lands in depend on `m` alone. - const auto run_morsel = [&](std::size_t m) { - const std::size_t begin = m * grain; - const std::size_t end = std::min(rows, begin + grain); - if (begin < end) { - accumulate_ungrouped_range_impl(agg_entries, begin, end, &partials[(m * n_aggs_)], - ung_scratch.data() + (m * scratch_stride_)); - } - }; - const std::size_t threads = - std::min(morsels, exec_ != nullptr ? exec_->compute_budget() : std::size_t{1}); - // Submitting from a pool thread would deadlock (`WorkerPool::submit` - // aborts rather than allow it), and one worker gains nothing from a - // round trip, so both run the morsels here. The arithmetic is unchanged - // either way. - if (threads >= 2 && !on_worker_pool_thread()) { - auto& pool = process_worker_pool(); - std::atomic cursor{0}; - auto batch = pool.submit(threads, [&](std::size_t) { - while (true) { - const std::size_t m = cursor.fetch_add(1, std::memory_order_relaxed); - if (m >= morsels) { - return; - } - run_morsel(m); - } - }); - batch.wait(); - if (exec_ != nullptr && exec_->parallel_stats != nullptr) { - exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); - } - } else { - for (std::size_t m = 0; m < morsels; ++m) { - run_morsel(m); - } - } - for (std::size_t m = 0; m < morsels; ++m) { - for (std::size_t a = 0; a < n_aggs_; ++a) { - agg_combine(dst[a], partials[(m * n_aggs_) + a], plan_[a].func, plan_[a].kind, - scratch_stride_ == 0 ? nullptr : scratch_for(0, a), - scratch_stride_ == 0 - ? nullptr - : ung_scratch.data() + (m * scratch_stride_) + scratch_offset_[a]); - } - } - return std::nullopt; - } - - /// How many row-morsels to split a global aggregate into; 1 = stay serial. - [[nodiscard]] auto ungrouped_morsels(std::size_t rows) const -> std::size_t { - // Deliberately NOT gated on `exec_->can_fan_out()`, the thread budget, or - // whether this runs on a pool thread. Those decide who EXECUTES the - // morsels, not how the range is cut, and a float reduction's result - // depends on where it is cut. Keeping the cut a function of the data - // alone is what makes one worker, eight workers, a serial run and a - // nested run agree bit for bit. The caller runs the morsels inline - // when it cannot fan out. - for (std::size_t a = 0; a < n_aggs_; ++a) { - if (!agg_is_combinable(plan_[a].func)) { - return 1; // Skew/Kurtosis: no partial merge, stay serial. - } - // A non-numeric First/Last keeps its value in `text_store_`, which - // agg_combine cannot reach and workers must not write concurrently. - if (plan_[a].kind != ExprType::Int && plan_[a].kind != ExprType::Double) { - return 1; - } - } - // The partition is a function of the ROW COUNT ALONE — deliberately - // not of the thread count. A float reduction's result depends on where - // the range is cut, so deriving morsels from the pool size would make - // `sum`/`std` answers differ between a 4-core box and a 24-core one, - // and differ again under `--threads`. Keyed on rows, the answer depends - // only on the data: same input, same result, any machine, any schedule. - // - // Morsels are large because a reduction's per-row cost is constant — - // equal ranges finish together, so unlike a filter there is no - // imbalance to hedge against and every extra morsel is pure dispatch - // and merge overhead. The cap bounds the partial array. - constexpr std::size_t kMinRowsPerMorsel = 65536; - constexpr std::size_t kMaxMorsels = 64; - return std::clamp(rows / kMinRowsPerMorsel, 1, kMaxMorsels); - } - - auto build_output_chunk() -> std::expected, std::string> { - // Owned-partition mode: the deferred first-occurrence merge runs - // exactly once, here, right before the ordinary emission logic below - // -- which then runs completely unmodified, reading the same - // pair_order_/flat_slots_/n_groups_ it always has. - if (owned_mode_) { - finalize_owned_active(); - if (owned_async_error_.has_value()) { - return std::unexpected(*owned_async_error_); - } - } - - Chunk out; - out.columns.reserve(group_by_->size() + aggregations_->size()); - - if (!initialized_) { - // No input rows at all — emit a chunk with empty columns of - // the expected schema where possible. Without any chunk we - // have no types to build group columns; return an empty - // optional so the sink finalizes an empty table. - return std::optional{}; - } - - for (std::size_t i = 0; i < group_by_->size(); ++i) { - out.add_column((*group_by_)[i].name, make_empty_like(group_templates_[i])); - } - for (std::size_t i = 0; i < aggregations_->size(); ++i) { - const auto& agg = (*aggregations_)[i]; - ColumnValue column; - switch (agg.func) { - case ir::AggFunc::Count: - column = Column{}; - break; - case ir::AggFunc::Mean: - case ir::AggFunc::Stddev: - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - column = Column{}; - break; - case ir::AggFunc::Sum: - case ir::AggFunc::Min: - case ir::AggFunc::Max: - if (plan_[i].kind == ExprType::Double) { - column = Column{}; - } else { - column = Column{}; - } - break; - case ir::AggFunc::First: - case ir::AggFunc::Last: - if (plan_[i].kind == ExprType::Double) { - column = Column{}; - } else if (plan_[i].kind == ExprType::Int) { - column = Column{}; - } else if (plan_[i].categorical) { - column = Column{}; - } else { - column = Column{}; - } - break; - default: - return std::unexpected( - "ChunkedAggregateOperator: unsupported agg in build_output"); - } - out.add_column(agg.alias, std::move(column)); - } - - for (std::size_t i = 0; i < out.columns.size(); ++i) { - std::visit([&](auto& c) { c.reserve(n_groups_); }, out.mutable_column(i)); - } - - std::vector agg_validity(aggregations_->size()); - std::vector track_validity(aggregations_->size(), 0U); - for (std::size_t i = 0; i < aggregations_->size(); ++i) { - if (chunked_agg_tracks_validity(plan_[i].func)) { - track_validity[i] = 1U; - agg_validity[i].reserve(n_groups_); - } - } - - // The null group's key cell carries the type's zero value plus a clear - // validity bit. Only the generic Key path can produce one — the cat/str - // fast paths above are only taken for key columns with no nulls. - std::vector key_validity(group_by_->size()); - std::uint64_t any_null_keys = 0; - if (!cat_fast_path_ && !str_fast_path_ && !int_fast_path_ && !pair_int_fast_path_) { - for (const auto& key : group_order_) { - any_null_keys |= key.null_mask; - } - if (any_null_keys != 0) { - for (auto& bitmap : key_validity) { - bitmap.assign(n_groups_, true); - } - } - } - - const auto push_int_key = [](ColumnValue& col, IntKeyKind kind, std::int64_t raw) { - switch (kind) { - case IntKeyKind::Int64: - std::get>(col).push_back(raw); - return; - case IntKeyKind::Date: - std::get>(col).push_back(Date{static_cast(raw)}); - return; - case IntKeyKind::Ts: - std::get>(col).push_back(Timestamp{raw}); - return; - case IntKeyKind::Cat: - // The output column is `make_empty_like` of the input, so it - // shares the input's dictionary and the stored code resolves - // against it. - std::get>(col).push_code( - static_cast::code_type>(raw)); - return; - } - }; - - const AggSlotCore* fs = flat_slots_.data(); - - // Emission is column-major, one output column per task: every column is - // a separate buffer written by exactly one worker, so no two tasks touch - // the same bytes and the emitted order is the group order regardless of - // which worker got which column. - const auto emit_key_column = [&](std::size_t ci) { - ColumnValue& col = out.mutable_column(ci); - if (cat_fast_path_) { - auto& cat_col = std::get>(col); - const std::size_t n_keys = group_by_->size(); - if (n_keys == 1) { - for (std::size_t g = 0; g < n_groups_; ++g) { - cat_col.push_code(cat_order_[g]); - } - } else { - for (std::size_t g = 0; g < n_groups_; ++g) { - cat_col.push_code(multi_cat_codes_flat_[(g * n_keys) + ci]); - } - } - } else if (str_fast_path_) { - auto& str_col = std::get>(col); - for (std::size_t g = 0; g < n_groups_; ++g) { - str_col.push_back(str_order_[g]); - } - } else if (int_fast_path_) { - for (std::size_t g = 0; g < n_groups_; ++g) { - push_int_key(col, int_key_kind_, int_order_[g]); - } - } else if (pair_int_fast_path_) { - const IntKeyKind kind = ci == 0 ? int_key_kind_ : int_key_kind_b_; - for (std::size_t g = 0; g < n_groups_; ++g) { - push_int_key(col, kind, ci == 0 ? pair_order_[g].first : pair_order_[g].second); - } - } else { - for (std::size_t g = 0; g < n_groups_; ++g) { - const Key& key = group_order_[g]; - if (ci >= key.values.size()) { - continue; - } - append_scalar(col, key.values[ci]); - if (any_null_keys != 0 && ci < kMaxKeyColumns && - (key.null_mask & (std::uint64_t{1} << ci)) != 0) { - key_validity[ci].set(g, false); - } - } - } - }; - - const auto emit_agg_column = [&](std::size_t i) { - ColumnValue& column = out.mutable_column(group_by_->size() + i); - const bool tracks_validity = track_validity[i] != 0U; - for (std::size_t g = 0; g < n_groups_; ++g) { - const AggSlotCore& slot = fs[(g * n_aggs_) + i]; - if (tracks_validity) { - agg_validity[i].push_back(chunked_agg_valid(plan_[i].func, slot)); - } - switch (plan_[i].func) { - case ir::AggFunc::Count: - append_scalar(column, slot.count); - break; - case ir::AggFunc::Mean: - append_scalar(column, - slot.count == 0 - ? 0.0 - : slot.double_value / static_cast(slot.count)); - break; - case ir::AggFunc::Sum: - case ir::AggFunc::Min: - case ir::AggFunc::Max: - if (plan_[i].kind == ExprType::Double) { - append_scalar(column, slot.double_value); - } else { - append_scalar(column, slot.int_value); - } - break; - case ir::AggFunc::Stddev: - append_scalar(column, agg_finalize_stddev(slot, scratch_for(g, i)[0])); - break; - case ir::AggFunc::Skew: - append_scalar(column, agg_finalize_skew(slot, scratch_for(g, i)[0], - scratch_for(g, i)[1])); - break; - case ir::AggFunc::Kurtosis: - append_scalar(column, agg_finalize_kurtosis(slot, scratch_for(g, i)[0], - scratch_for(g, i)[2])); - break; - case ir::AggFunc::First: - case ir::AggFunc::Last: - if (plan_[i].kind == ExprType::Double) { - append_scalar(column, slot.double_value); - } else if (plan_[i].kind == ExprType::Int) { - append_scalar(column, slot.int_value); - } else { - append_scalar(column, text_store_[(g * n_aggs_) + i]); - } - break; - default: - break; - } - } - }; - - const std::size_t n_out_columns = out.columns.size(); - const auto emit_column = [&](std::size_t c) { - if (c < group_by_->size()) { - emit_key_column(c); - } else { - emit_agg_column(c - group_by_->size()); - } - }; - - // A one-task budget would pay the pool round trip for work the calling - // thread is about to do anyway, so it stays serial. - auto& pool = process_worker_pool(); - const std::size_t threads = - std::min(n_out_columns, exec_ != nullptr ? exec_->compute_budget() : pool.size()); - if (exec_ != nullptr && exec_->can_fan_out() && !on_worker_pool_thread() && threads > 1 && - n_groups_ >= exec_->parallel_min_rows) { - std::atomic cursor{0}; - auto batch = pool.submit(threads, [&](std::size_t) { - while (true) { - const std::size_t c = cursor.fetch_add(1, std::memory_order_relaxed); - if (c >= n_out_columns) { - return; - } - emit_column(c); - } - }); - batch.wait(); - if (exec_->parallel_stats != nullptr) { - exec_->parallel_stats->parallel_fields.fetch_add(1, std::memory_order_relaxed); - } - } else { - for (std::size_t c = 0; c < n_out_columns; ++c) { - emit_column(c); - } - } - - for (std::size_t i = 0; i < aggregations_->size(); ++i) { - if (track_validity[i] == 0U || agg_validity[i].empty()) { - continue; - } - bool has_null = false; - for (std::size_t r = 0; r < agg_validity[i].size(); ++r) { - if (!agg_validity[i][r]) { - has_null = true; - break; - } - } - if (has_null) { - out.columns[group_by_->size() + i].validity = std::move(agg_validity[i]); - } - } - - for (std::size_t ci = 0; ci < group_by_->size() && ci < kMaxKeyColumns; ++ci) { - if ((any_null_keys & (std::uint64_t{1} << ci)) != 0) { - out.columns[ci].validity = std::move(key_validity[ci]); - } - } - - return std::optional{std::move(out)}; - } - - struct SlotPlan { - ir::AggFunc func = ir::AggFunc::Sum; - ExprType kind = ExprType::Int; - // Only meaningful when kind == String: disambiguates Column - // from Column for First/Last output construction, since - // expr_type_for_column collapses both to ExprType::String. - bool categorical = false; - /// Extra per-GROUP state this aggregate needs, in doubles. Zero for - /// almost everything, which is the point: state that only one - /// aggregate wants must not sit in AggSlotCore, where it would cost - /// every group of every query. Skew/Kurtosis declare 2 (the third and - /// fourth central moments); a future aggregate declares whatever it - /// needs without touching the slot. - std::uint32_t scratch_doubles = 0; - }; - - struct CatKey { - std::vector::code_type> codes; - auto operator==(const CatKey& o) const noexcept -> bool { return codes == o.codes; } - }; - - // Transparent hash/eq: lets `str_index_.find(string_view)` skip the - // allocation of a temporary std::string on every probe. - struct StrViewHash { - using is_transparent = void; - auto operator()(std::string_view s) const noexcept -> std::size_t { - return robin_hood::hash_bytes(s.data(), s.size()); - } - auto operator()(const std::string& s) const noexcept -> std::size_t { - return robin_hood::hash_bytes(s.data(), s.size()); - } - }; - struct StrViewEq { - using is_transparent = void; - auto operator()(const std::string& a, const std::string& b) const noexcept -> bool { - return a == b; - } - auto operator()(const std::string& a, std::string_view b) const noexcept -> bool { - return std::string_view(a) == b; - } - auto operator()(std::string_view a, const std::string& b) const noexcept -> bool { - return a == std::string_view(b); - } - }; - struct CatKeyHash { - auto operator()(const CatKey& k) const noexcept -> std::size_t { - std::size_t h = 0; - for (auto c : k.codes) { - h ^= robin_hood::hash::code_type>{}(c) + 0x9e3779b9 + - (h << 6U) + (h >> 2U); - } - return h; - } - }; - - OperatorPtr child_; - const std::vector* group_by_; - const std::vector* aggregations_; - const ExecutionContext* exec_; - bool emitted_ = false; - - bool initialized_ = false; - bool cat_fast_path_ = false; - bool str_fast_path_ = false; - std::size_t n_aggs_ = 0; - std::size_t n_groups_ = 0; - std::vector plan_; - std::vector group_templates_; - std::vector discovery_first_eligible_; - - /// Per-group scratch for aggregates that declared `scratch_doubles`, - /// laid out group-major: `scratch_[gid * scratch_stride_ + offset[agg]]`. - /// Group-major so several small consumers in one group share a cache line. - /// Stays EMPTY when no aggregate asks for any — the same "pay only if used" - /// rule as text_store_, and `double` keeps it trivially copyable so growth - /// is a memcpy. - std::vector scratch_; - std::size_t scratch_stride_ = 0; - std::vector scratch_offset_; - - /// Boxed First/Last values for non-numeric columns, parallel to - /// `flat_slots_` and indexed identically. Stays EMPTY — no allocation, no - /// ScalarValue construction — for an all-numeric query, which is why the - /// slot itself can be a POD. - std::vector text_store_; - - // Flat accumulator storage: n_groups_ × n_aggs_ contiguous AggSlotCores. - SlotArray flat_slots_; - - // Reusable per-chunk gids buffer to avoid repeated heap allocations. - std::vector gids_buf_; - - // Generic path (non-Categorical group keys). - KeyRowIndex key_index_; - std::vector group_order_; - - // Sentinel for "no group assigned yet" in the dense index arrays. - static constexpr std::uint32_t kNoGid = std::numeric_limits::max(); - // Cartesian cell-space size below which multi-key grouping uses a dense - // array (one load per row) instead of hashing. 4M cells = 16 MB of u32. - static constexpr std::uint64_t kDenseCellLimit = 4'000'000ULL; - - // Single-Categorical fast path: code → gid via direct array (codes are a - // dense [0, dict_size) index, so no hashing is needed). - using cat_code = Column::code_type; - std::vector cat_dense_gid_; - std::vector cat_order_; - - // Multi-Categorical fast path: cell-encoded. Dense array while the cell - // space stays under kDenseCellLimit; spills to the hash map otherwise. - bool multi_dense_ = true; - std::vector multi_cat_cell_dense_; - std::vector multi_cat_slots_; // open addressing on the code tuple: gid + 1 - std::vector::code_type> multi_cat_codes_flat_; // n_groups_ × n_keys - std::vector multi_cat_strides_; // last-seen strides for rebuild detection - - // Single-string-key fast path. - robin_hood::unordered_flat_map str_index_; - std::vector str_order_; - - // Single fixed-width-integer-key fast path (int64 / Date / Timestamp, no - // nulls): a direct value -> gid map, no owned Key per group. `group by ` is one of the most common shapes, and the generic path was building a - // heap-allocated Key per group for it (117k allocations on TPC-H q02's - // 117k-group min). - bool int_fast_path_ = false; - IntKeyKind int_key_kind_ = IntKeyKind::Int64; - robin_hood::unordered_flat_map int_index_; - std::vector int_order_; ///< group keys, as raw integers, in first-seen order - - // Two fixed-width-integer keys are packed into a two-word composite key - // and grouped exactly as one integer key: `(l_partkey, l_suppkey)` on - // TPC-H q20's ~800k groups otherwise falls to the generic `Key` path, which - // boxes a ScalarValue-vector Key per group and compares it field-by-field on - // every probe. Keeping two 64-bit values is injective with no knowledge of - // their domains, so this is always exact and portable to MSVC. - bool pair_int_fast_path_ = false; - /// Three or more key columns, all fixed-width and packable. Sets no other - /// fast-path flag, so the output path treats it as the generic key case. - bool packed_fast_path_ = false; - PackedKeyEncoder encoder_; - PackedGroups> packed64_; - PackedGroups> packed128_; - PackedGroups> packed256_; - IntKeyKind int_key_kind_b_ = IntKeyKind::Int64; - struct PairIntKey { - std::uint64_t first = 0; - std::uint64_t second = 0; - - [[nodiscard]] friend auto operator==(const PairIntKey&, const PairIntKey&) - -> bool = default; - }; - struct PairIntKeyHash { - auto operator()(const PairIntKey& key) const noexcept -> std::size_t { - std::uint64_t h = key.first * 0x9e3779b97f4a7c15ULL; - h ^= key.second + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); - return static_cast(h); - } - }; - robin_hood::unordered_flat_map pair_index_; - std::vector> pair_order_; - /// Parallel group discovery (see `try_discover_partitioned`). `rows_seen_` - /// makes a group's first-row index global across chunks, which is what the - /// first-occurrence numbering is merged on. - std::vector> pair_partitions_; - std::vector>> int_partitions_; - std::vector> str_partitions_; - std::vector part_of_row_; - std::vector scatter_rows_; - std::uint64_t rows_seen_ = 0; - - // --- Partition-owned aggregation (plans/parallelism-overview.md "stream - // multi-key joins" successor, step 2): the PairIntKey path only, admitted - // by `try_owned_pair`'s gates. `IBEX_DISABLE_OWNED_PAIR_AGG=1` is its kill - // switch. - - /// A partition's group discovery AND its own final aggregate state -- - /// no global gid, no global `flat_slots_` entry, until - /// `finalize_owned_pair` walks every partition once at - /// final emission to restore first-occurrence order (plans doc: "moves - /// that merge... does not eliminate the ordered merge itself"). - template > - struct OwnedPartition { - robin_hood::unordered_flat_map index; - std::vector keys; - std::vector first_rows; - std::vector slots; ///< n_local_groups * n_aggs_ - }; - std::vector> owned_pair_partitions_; - std::vector>> owned_int_partitions_; - // q18's Polars-style sink: per-chunk 4096-slot hot reducers publish compact - // pre-aggregates asynchronously; one final owner task per cold partition - // builds the persistent maps. Jobs are declared before the task group so - // reverse destruction joins every capture before releasing its storage. - std::vector> owned_async_jobs_; - std::optional owned_async_group_; - std::optional owned_async_error_; - std::size_t owned_async_part_count_ = 0; - bool owned_async_hot_mode_ = false; - std::vector owned_ordered_run_keys_; - std::vector owned_ordered_run_counts_; - bool owned_ordered_run_mode_ = false; - bool owned_ordered_runs_nondecreasing_ = true; - /// Set once this operator has committed to owned-partition mode. Per the - /// plan's safety note, only ever ADMITTED before any other discovery path - /// (serial or `try_discover_partitioned`) has created a group -- widening - /// to seed/migrate an in-progress run is out of scope for this prototype. - bool owned_mode_ = false; - bool owned_finalized_ = false; - std::uint64_t owned_rows_seen_ = 0; - /// Set once `try_discover_partitioned` has run; see the gate there for why - /// a later chunk may then never fall back to the serial loop. - bool partitioned_active_ = false; - /// Input rows this operator has been offered across every chunk, which is - /// what the partition gate measures. Distinct from `rows_seen_`, which - /// counts only rows the partitioned path itself consumed and exists to give - /// group first-rows a global base. - std::size_t rows_offered_ = 0; - /// The `partition` and `finalize` fan-out policies, resolved by - /// `build_physical_aggregate` (src/runtime/PARALLELISM.md). The operator - /// reads `par_..{decline, worker_cap}` for fan-out permission and the - /// worker ceiling; it keeps only what the plan cannot know -- nesting - /// (`on_worker_pool_thread`), the data-derived partition/run terms of each - /// cap, and the strategy-specific admission floors (`kPairOwnedMinRows`, the - /// ordered-run `1U << 16U`, `parallel_min_rows`). - physical::AggregateParallelism par_{}; - /// Both keys are 32 bits wide (Categorical code / Date), so the composite - /// packs into 64 bits and probes `int_index_` instead of `pair_index_`. - /// The two paths are mutually exclusive, so sharing that map is safe. - bool pair_packs_u64_ = false; - /// Flat cell -> gid for a packed pair whose domains are small enough to - /// enumerate. Cells are numbered from the mins below, so widening them - /// rebuilds the array. - std::vector pair_dense_gid_; - std::int64_t pair_dense_a_min_ = 0; - std::int64_t pair_dense_a_max_ = 0; - std::int64_t pair_dense_b_min_ = 0; - std::int64_t pair_dense_b_max_ = 0; - std::uint64_t pair_dense_b_span_ = 0; - bool pair_dense_active_ = false; -}; - -/// Replays one buffered chunk ahead of the rest of a child stream. Used by -/// ChunkedSortedAggregateOperator to hand the already-pulled first chunk back -/// to a fallback operator without losing it. -class PrependChunkOperator final : public Operator { - public: - PrependChunkOperator(Chunk first, OperatorPtr rest) - : first_(std::move(first)), rest_(std::move(rest)) {} - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (!emitted_first_) { - emitted_first_ = true; - return std::optional{std::move(first_)}; - } - return rest_->next(); - } - - private: - Chunk first_; - OperatorPtr rest_; - bool emitted_first_ = false; -}; - -/// Streaming aggregate for input already sorted on the group-by keys. -/// -/// When the child's chunks declare an `ordering` whose leading keys cover the -/// group_by columns, every group's rows are contiguous in the stream. We then -/// keep accumulators for only the *current* group, emit each group as soon as -/// its run ends, and produce output already sorted by the group keys. Peak -/// memory is O(one group + one output chunk) instead of O(all groups), and -/// there is no hashing — group changes are detected by a typed equality scan. -/// -/// Eligibility is decided from the first non-empty chunk. If the input is not -/// sorted on the group_by keys (no `ordering`, or it doesn't cover them, or a -/// group key is nullable), the operator transparently falls back to the -/// hash-based ChunkedAggregateOperator by replaying the already-pulled chunk -/// ahead of the remaining child. The supported agg subset matches -/// ChunkedAggregateOperator (Count/Sum/Min/Max/Mean on numeric columns); -/// build_operator only routes that subset here. -class ChunkedSortedAggregateOperator final : public Operator { - public: - ChunkedSortedAggregateOperator(OperatorPtr child, const std::vector* group_by, - const std::vector* aggregations, - const ExecutionContext& exec, - physical::AggregateParallelism par = {}) - : child_(std::move(child)), - group_by_(group_by), - aggregations_(aggregations), - exec_(&exec), - par_(par) {} - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (fallback_) { - return fallback_->next(); - } - if (!decided_) { - auto decided = decide_strategy(); - if (!decided.has_value()) { - return std::unexpected(std::move(decided.error())); - } - if (fallback_) { - return fallback_->next(); - } - } - return next_sorted(); - } - - private: - struct SlotPlan { - ir::AggFunc func = ir::AggFunc::Sum; - ExprType kind = ExprType::Int; - }; - - // Pull chunks until the first non-empty one, then choose sorted vs fallback. - auto decide_strategy() -> std::expected { - decided_ = true; - Chunk first; - bool have = false; - std::optional schema_only; - while (true) { - auto chunk_res = child_->next(); - if (!chunk_res.has_value()) { - return std::unexpected(std::move(chunk_res.error())); - } - if (!chunk_res.value().has_value()) { - break; // EOF before any rows - } - if (chunk_res.value()->rows() == 0) { - // Empty, but it still carries the input's columns and their - // types. Keep the first one in case no chunk ever has rows. - if (!schema_only.has_value() && !chunk_res.value()->columns.empty()) { - schema_only = std::move(*chunk_res.value()); - } - continue; - } - first = std::move(*chunk_res.value()); - have = true; - break; - } - if (!have) { - // Every row was filtered away upstream. Emitting nothing would emit - // no schema either, and the result would materialize with no columns - // at all — so a downstream join looking for its key, or a filter for - // the value it compares, would fail with "unknown column" on what is - // really just an empty input. The hash operator derives its output - // columns from the input's types, so hand it the empty chunk and let - // it produce a properly-shaped empty result. - if (schema_only.has_value()) { - fallback_ = std::make_unique( - std::make_unique(std::move(*schema_only), - std::move(child_)), - group_by_, aggregations_, *exec_, par_); - return {}; - } - done_ = true; - input_eof_ = true; - return {}; - } - if (!sorted_on_group_by(first) || needs_hash_fallback(first)) { - fallback_ = std::make_unique( - std::make_unique(std::move(first), std::move(child_)), - group_by_, aggregations_, *exec_, par_); - return {}; - } - if (auto err = init_plan(first)) { - return std::unexpected(*err); - } - if (auto err = consume(first)) { - return std::unexpected(*err); - } - return {}; - } - - // The input is grouped-contiguous iff the first |group_by| ordering keys - // are exactly the group_by columns (as a set; direction and intra-prefix - // order don't matter for contiguity). Nullable group keys fall back, since - // the streaming key compare ignores validity. - [[nodiscard]] auto sorted_on_group_by(const Chunk& chunk) const -> bool { - if (group_by_->empty()) { - return false; // global aggregate: let the hash path handle it - } - if (!chunk.ordering().has_value() || chunk.ordering()->size() < group_by_->size()) { - return false; - } - const auto& ordering = *chunk.ordering(); - for (std::size_t i = 0; i < group_by_->size(); ++i) { - bool in_group = false; - for (const auto& g : *group_by_) { - if (g.name == ordering[i].name) { - in_group = true; - break; - } - } - if (!in_group) { - return false; - } - } - return std::ranges::all_of(*group_by_, [&chunk](const auto& g) { - const ColumnEntry* entry = find_entry(chunk, g.name); - return entry != nullptr && !entry->validity.has_value(); - }); - } - - static auto find_entry(const Chunk& chunk, const std::string& name) -> const ColumnEntry* { - for (const auto& e : chunk.columns) { - if (e.name == name) { - return &e; - } - } - return nullptr; - } - - // Non-numeric First/Last (string/categorical) has no group-at-a-time - // implementation here — route it to the hash operator, which handles any - // type. Numeric First/Last streams natively (see accumulate_typed). - [[nodiscard]] auto needs_hash_fallback(const Chunk& first) const -> bool { - return std::ranges::any_of(*aggregations_, [&](const ir::AggSpec& agg) { - if (agg.func != ir::AggFunc::First && agg.func != ir::AggFunc::Last) { - return false; - } - const ColumnEntry* entry = find_entry(first, agg.column.name); - if (entry == nullptr) { - return false; // reported as a proper error by init_plan - } - const ExprType kind = expr_type_for_column(*entry->column); - return kind != ExprType::Int && kind != ExprType::Double; - }); - } - - auto init_plan(const Chunk& first) -> std::optional { - n_aggs_ = aggregations_->size(); - plan_.resize(n_aggs_); - for (std::size_t i = 0; i < n_aggs_; ++i) { - const auto& agg = (*aggregations_)[i]; - plan_[i].func = agg.func; - if (agg.func == ir::AggFunc::Count) { - plan_[i].kind = ExprType::Int; - continue; - } - const ColumnEntry* entry = find_entry(first, agg.column.name); - if (entry == nullptr) { - return "aggregate column not found: " + agg.column.name; - } - const ExprType kind = expr_type_for_column(*entry->column); - if (kind != ExprType::Int && kind != ExprType::Double) { - return "ChunkedSortedAggregateOperator: non-numeric aggregation not supported"; - } - plan_[i].kind = kind; - } - key_templates_.clear(); - key_templates_.reserve(group_by_->size()); - for (const auto& g : *group_by_) { - key_templates_.push_back(make_empty_like(*find_entry(first, g.name)->column)); - } - track_validity_.assign(n_aggs_, 0U); - for (std::size_t i = 0; i < n_aggs_; ++i) { - track_validity_[i] = chunked_agg_tracks_validity(plan_[i].func) ? 1U : 0U; - } - // Capture the leading ordering keys so emitted chunks can advertise the - // group-sorted order they preserve (lets a downstream `order` skip work). - if (first.ordering().has_value()) { - out_ordering_.assign( - first.ordering()->begin(), - first.ordering()->begin() + static_cast(group_by_->size())); - } - cur_slots_.assign(n_aggs_, AggSlotCore{}); - cur_scratch_.assign(n_aggs_ * kMomentScratch, 0.0); - reset_output(); - return std::nullopt; - } - - void reset_output() { - out_columns_.clear(); - out_columns_.reserve(group_by_->size() + n_aggs_); - for (std::size_t i = 0; i < group_by_->size(); ++i) { - ColumnEntry entry; - entry.name = (*group_by_)[i].name; - entry.column = std::make_shared(make_empty_like(key_templates_[i])); - std::visit([&](auto& c) { c.reserve(kEmitThreshold); }, *entry.column); - out_columns_.push_back(std::move(entry)); - } - for (std::size_t i = 0; i < n_aggs_; ++i) { - ColumnValue column; - switch (plan_[i].func) { - case ir::AggFunc::Count: - column = Column{}; - break; - case ir::AggFunc::Mean: - case ir::AggFunc::Stddev: - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - column = Column{}; - break; - default: // Sum / Min / Max - column = plan_[i].kind == ExprType::Double - ? ColumnValue{Column{}} - : ColumnValue{Column{}}; - break; - } - std::visit([&](auto& c) { c.reserve(kEmitThreshold); }, column); - ColumnEntry entry; - entry.name = (*aggregations_)[i].alias; - entry.column = std::make_shared(std::move(column)); - out_columns_.push_back(std::move(entry)); - } - out_validity_.assign(n_aggs_, ValidityBitmap{}); - for (std::size_t i = 0; i < n_aggs_; ++i) { - if (track_validity_[i] != 0U) { - out_validity_[i].reserve(kEmitThreshold); - } - } - pending_rows_ = 0; - } - - // Drive input until we have a full output batch or hit EOF, then emit. - auto next_sorted() -> std::expected, std::string> { - if (done_) { - return std::optional{}; - } - while (!input_eof_ && pending_rows_ < kEmitThreshold) { - auto chunk_res = child_->next(); - if (!chunk_res.has_value()) { - return std::unexpected(std::move(chunk_res.error())); - } - if (!chunk_res.value().has_value()) { - input_eof_ = true; - break; - } - if (chunk_res.value()->rows() == 0) { - continue; - } - if (auto err = consume(*chunk_res.value())) { - return std::unexpected(*err); - } - } - if (input_eof_ && open_) { - close_group(); - open_ = false; - } - if (pending_rows_ == 0) { - done_ = true; - return std::optional{}; - } - Chunk out = take_pending(); - if (input_eof_) { - done_ = true; - } - return std::optional{std::move(out)}; - } - - // Fold one chunk into the streaming state. Rows are scanned as runs of - // equal group keys; each run is accumulated columnwise into the open - // group's slots, and a group-key change closes the open group. - auto consume(const Chunk& chunk) -> std::optional { - std::vector key_cols; - key_cols.reserve(group_by_->size()); - for (const auto& g : *group_by_) { - const ColumnEntry* entry = find_entry(chunk, g.name); - if (entry == nullptr) { - return "group-by column not found: " + g.name; - } - key_cols.push_back(entry->column.get()); - } - std::vector agg_entries(n_aggs_, nullptr); - for (std::size_t i = 0; i < n_aggs_; ++i) { - if (plan_[i].func == ir::AggFunc::Count) { - continue; - } - const ColumnEntry* entry = find_entry(chunk, (*aggregations_)[i].column.name); - if (entry == nullptr) { - return "aggregate column not found: " + (*aggregations_)[i].column.name; - } - if (expr_type_for_column(*entry->column) != plan_[i].kind) { - return "ChunkedSortedAggregateOperator: aggregate column type changed across " - "chunks"; - } - agg_entries[i] = entry; - } - - const std::size_t rows = chunk.rows(); - std::size_t r = 0; - while (r < rows) { - if (!open_) { - start_group(key_cols, r); - } else if (!row_matches_open(key_cols, r)) { - close_group(); - start_group(key_cols, r); - } - std::size_t e = r + 1; - while (e < rows && cells_equal(key_cols, r, e)) { - ++e; - } - accumulate_range(agg_entries, r, e); - r = e; - } - return std::nullopt; - } - - void start_group(const std::vector& key_cols, std::size_t row) { - open_key_.clear(); - open_key_.reserve(key_cols.size()); - for (const auto* col : key_cols) { - open_key_.push_back(scalar_from_column(*col, row)); - } - std::ranges::fill(cur_slots_, AggSlotCore{}); - std::ranges::fill(cur_scratch_, 0.0); - open_ = true; - } - - // Whether `row` continues the currently open group. Only called at run - // anchors (group boundaries and chunk starts), so the scalar build is - // paid per group, not per row. - [[nodiscard]] auto row_matches_open(const std::vector& key_cols, - std::size_t row) const -> bool { - for (std::size_t i = 0; i < key_cols.size(); ++i) { - if (scalar_from_column(*key_cols[i], row) != open_key_[i]) { - return false; - } - } - return true; - } - - static auto cell_equal(const ColumnValue& col, std::size_t a, std::size_t b) -> bool { - return std::visit( - [&](const auto& c) -> bool { - using ColT = std::decay_t; - if constexpr (std::is_same_v>) { - return c.code_at(a) == c.code_at(b); - } else if constexpr (std::is_same_v>) { - return c[a].days == c[b].days; - } else if constexpr (std::is_same_v>) { - return c[a].nanos == c[b].nanos; - } else { - return c[a] == c[b]; - } - }, - col); - } - - [[nodiscard]] static auto cells_equal(const std::vector& key_cols, - std::size_t a, std::size_t b) -> bool { - return std::ranges::all_of(key_cols, - [a, b](const auto* col) { return cell_equal(*col, a, b); }); - } - - // Accumulate the contiguous row range [start, end) — all one group — into - // the open group's slots, branch-hoisted per aggregation. - void accumulate_range(const std::vector& agg_entries, std::size_t start, - std::size_t end) { - for (std::size_t i = 0; i < n_aggs_; ++i) { - AggSlotCore& slot = cur_slots_[i]; - if (plan_[i].func == ir::AggFunc::Count) { - slot.count += static_cast(end - start); - continue; - } - const auto& entry = *agg_entries[i]; - const bool has_nulls = entry.validity.has_value(); - if (plan_[i].kind == ExprType::Double) { - const double* data = std::get>(*entry.column).data(); - accumulate_typed(slot, &cur_scratch_[i * kMomentScratch], plan_[i].func, data, - entry, has_nulls, start, end); - } else { - const std::int64_t* data = std::get>(*entry.column).data(); - accumulate_typed(slot, &cur_scratch_[i * kMomentScratch], plan_[i].func, data, - entry, has_nulls, start, end); - } - } - } - - template - /// `scratch` is this aggregate's per-group scratch (2 doubles for the - /// higher moments); it stays a parameter so this helper remains static and - /// has no reach into operator state. - static void accumulate_typed(AggSlotCore& slot, double* scratch, ir::AggFunc func, - const T* data, const ColumnEntry& entry, bool has_nulls, - std::size_t start, std::size_t end) { - const auto valid = [&](std::size_t row) { return !has_nulls || (*entry.validity)[row]; }; - switch (func) { - case ir::AggFunc::Sum: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row)) { - continue; - } - if constexpr (std::is_same_v) { - slot.double_value += data[row]; - } else { - slot.int_value += data[row]; - } - slot.mark_present(); - } - break; - case ir::AggFunc::Mean: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row)) { - continue; - } - slot.double_value += static_cast(data[row]); - slot.count++; - } - break; - case ir::AggFunc::Min: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row)) { - continue; - } - if constexpr (std::is_same_v) { - slot.double_value = - slot.present() ? std::min(slot.double_value, data[row]) : data[row]; - } else { - slot.int_value = - slot.present() ? std::min(slot.int_value, data[row]) : data[row]; - } - slot.mark_present(); - } - break; - case ir::AggFunc::Max: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row)) { - continue; - } - if constexpr (std::is_same_v) { - slot.double_value = - slot.present() ? std::max(slot.double_value, data[row]) : data[row]; - } else { - slot.int_value = - slot.present() ? std::max(slot.int_value, data[row]) : data[row]; - } - slot.mark_present(); - } - break; - case ir::AggFunc::Stddev: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row)) { - continue; - } - agg_update_stddev(slot, scratch[0], static_cast(data[row])); - } - break; - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row)) { - continue; - } - agg_update_moments(slot, scratch[0], scratch[1], scratch[2], - static_cast(data[row])); - } - break; - case ir::AggFunc::First: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row) || slot.present()) { - continue; - } - if constexpr (std::is_same_v) { - slot.double_value = data[row]; - } else { - slot.int_value = data[row]; - } - slot.mark_present(); - } - break; - case ir::AggFunc::Last: - for (std::size_t row = start; row < end; ++row) { - if (!valid(row)) { - continue; - } - if constexpr (std::is_same_v) { - slot.double_value = data[row]; - } else { - slot.int_value = data[row]; - } - slot.mark_present(); - } - break; - default: - break; - } - } - - // Flush the open group's key + aggregate values into the output buffers. - void close_group() { - for (std::size_t i = 0; i < group_by_->size(); ++i) { - append_scalar(*out_columns_[i].column, open_key_[i]); - } - for (std::size_t i = 0; i < n_aggs_; ++i) { - ColumnValue& column = *out_columns_[group_by_->size() + i].column; - const AggSlotCore& slot = cur_slots_[i]; - if (track_validity_[i] != 0U) { - out_validity_[i].push_back(chunked_agg_valid(plan_[i].func, slot)); - } - switch (plan_[i].func) { - case ir::AggFunc::Count: - append_scalar(column, ScalarValue{slot.count}); - break; - case ir::AggFunc::Mean: - append_scalar( - column, ScalarValue{slot.count == 0 ? 0.0 - : slot.double_value / - static_cast(slot.count)}); - break; - case ir::AggFunc::Stddev: - append_scalar(column, ScalarValue{agg_finalize_stddev( - slot, cur_scratch_[i * kMomentScratch])}); - break; - case ir::AggFunc::Skew: - append_scalar(column, ScalarValue{agg_finalize_skew( - slot, cur_scratch_[i * kMomentScratch], - cur_scratch_[(i * kMomentScratch) + 1])}); - break; - case ir::AggFunc::Kurtosis: - append_scalar(column, ScalarValue{agg_finalize_kurtosis( - slot, cur_scratch_[i * kMomentScratch], - cur_scratch_[(i * kMomentScratch) + 2])}); - break; - default: // Sum / Min / Max - if (plan_[i].kind == ExprType::Double) { - append_scalar(column, ScalarValue{slot.double_value}); - } else { - append_scalar(column, ScalarValue{slot.int_value}); - } - break; - } - } - ++pending_rows_; - } - - auto take_pending() -> Chunk { - for (std::size_t i = 0; i < n_aggs_; ++i) { - if (track_validity_[i] == 0U || out_validity_[i].empty()) { - continue; - } - bool has_null = false; - for (std::size_t r = 0; r < out_validity_[i].size(); ++r) { - if (!out_validity_[i][r]) { - has_null = true; - break; - } - } - if (has_null) { - out_columns_[group_by_->size() + i].validity = std::move(out_validity_[i]); - } - } - Chunk out; - out.columns = std::move(out_columns_); - if (!out_ordering_.empty()) { - out.set_properties(TableProperties::sorted_by(out_ordering_)); - } - reset_output(); - return out; - } - - OperatorPtr child_; - const std::vector* group_by_; - const std::vector* aggregations_; - const ExecutionContext* exec_; - /// Forwarded verbatim to the hash `ChunkedAggregateOperator` fallback -- - /// the sorted stream itself has no fan-out point (it emits group-at-a-time). - physical::AggregateParallelism par_{}; - - bool decided_ = false; - bool done_ = false; - bool input_eof_ = false; - bool open_ = false; - OperatorPtr fallback_; - - static constexpr std::size_t kEmitThreshold = 8192; - - std::size_t n_aggs_ = 0; - std::vector plan_; - std::vector key_templates_; - std::vector track_validity_; - std::vector out_ordering_; - - // Open-group state. - std::vector cur_slots_; - /// Scratch for the group currently being streamed — 2 doubles per - /// aggregate that declared any (see SlotPlan::scratch_doubles). This - /// operator holds one group at a time, so it needs one group's worth. - /// [m2, m3, m4] per aggregate. Every moment aggregate needs m2 -- the - /// higher ones read it on each update -- so the stride is uniform rather - /// than per-function; this operator keeps one group's worth, not millions. - static constexpr std::size_t kMomentScratch = 3; - std::vector cur_scratch_; - std::vector open_key_; - - // Output buffers for closed groups awaiting emission. - std::vector out_columns_; - std::vector out_validity_; - std::size_t pending_rows_ = 0; -}; - -} // namespace - -auto materialize_operator(OperatorPtr op) -> std::expected { - MaterializeOperator sink{std::move(op)}; - return sink.run(); -} - -namespace { - -/// Defined next to `build_physical_join`; the join construction sites above it -/// (`inner_join_table`, the `IBEX_PROBE_MORSELS` probe POC) need it too. -auto resolved_join_parallelism(const ExecutionContext& exec) -> physical::JoinParallelism; - -/// The hash aggregate's `partition` and `finalize` fan-out policies, resolved -/// together. Defined next to `build_physical_aggregate`. -auto resolved_aggregate_parallelism(const ExecutionContext& exec) -> physical::AggregateParallelism; - -} // namespace - -auto distinct_table(const Table& input, const ExecutionContext& exec) - -> std::expected { - // I4 convergence: one implementation, reached through both shapes. The - // whole-table signature survives; the whole-table dedup loop does not. - // - // The source copy is shallow — a `Table` holds shared column handles — so - // this costs a vector of names and pointers, not the data. `distinct` on an - // empty column list still works: the operator passes such a chunk straight - // through, which is what the old `columns.empty()` special case did. - auto source = make_table_source(input); - // The whole table is one chunk, so its row count is exact -- the dedup - // phase's fan-out is fully decided here rather than on the first chunk. - physical::BreakerParallelism dedup_plan = physical::distinct_dedup_parallelism( - {.rows = input.rows(), .source = physical::RowEstimate::Source::ChildExact}); - const std::size_t pool_size = exec.can_fan_out() ? process_worker_pool().size() : 0; - physical::resolve_breaker_parallelism(dedup_plan, exec, pool_size); - return materialize_operator( - std::make_unique(std::move(source), dedup_plan)); -} - -auto is_streamable_inner_join(const ir::JoinNode& join) -> bool { - // `nulls equal` goes to the materialized join, which implements the policy. - // The streaming operators hash and probe on their own and would each need - // the same null tagging; sending the opt-in case to the one implementation - // that has it keeps a single definition of the semantics. - return join.kind() == ir::JoinKind::Inner && !join.predicate().has_value() && - join.keys().size() == 1 && join.null_match() == ir::NullMatch::Never && - !join.expect().asserts_anything() && join.take() == ir::MatchSelection::All; -} - -/// Two-fixed-width-int-key inner join (plans/parallelism-overview.md's -/// "stream multi-key joins" item; TPC-H q09's `lineitem` join is the -/// motivating case). Same structural gate as `is_streamable_inner_join` -/// plus a static schema check -- both keys, on both sides, must be provably -/// `Int64` -- so an ineligible pair (a string key, an unascribed/Unknown -/// schema) falls through to `build_binary_materializing_operator` exactly -/// as it does today, never into a code path that could fail at runtime. -/// `ChunkedInnerJoinOperator::initialize_pair` re-checks the actual runtime -/// column type regardless -- this is a routing optimization, not the sole -/// guarantee of correctness. -/// Whether every aggregation in `agg` can be computed incrementally, and so -/// streamed rather than materialized. -/// -/// Named and shared for the same reason the join gates were: the physical -/// planner has to relay this rather than restate it. Reimplementing a -/// multi-clause eligibility test in the planner is what made it wrong about -/// two-key joins within an hour of being written. -auto aggregate_is_streamable(const ir::AggregateNode& agg) -> bool { - return std::ranges::all_of(agg.aggregations(), [](const ir::AggSpec& spec) { - switch (spec.func) { - case ir::AggFunc::Count: - case ir::AggFunc::Sum: - case ir::AggFunc::Min: - case ir::AggFunc::Max: - case ir::AggFunc::Mean: - case ir::AggFunc::Stddev: - case ir::AggFunc::Skew: - case ir::AggFunc::Kurtosis: - case ir::AggFunc::First: - case ir::AggFunc::Last: - // First/Last: the operators themselves gate by column type - // (numeric, string, categorical stream; Date/Timestamp fall to - // the hash operator's error path -- unreachable in practice - // since aggregation on those types is rejected upstream of the - // chunked path entirely, same as every other agg func). - return true; - default: - // Median/Quantile need all values; Ewma is row-order coupled -- - // these stay on the materializing path. - return false; - } - }); -} - -auto is_streamable_semi_anti_join(const ir::JoinNode& join) -> bool { - return (join.kind() == ir::JoinKind::Semi || join.kind() == ir::JoinKind::Anti) && - !join.predicate().has_value() && join.keys().size() == 1 && - join.null_match() == ir::NullMatch::Never && !join.expect().asserts_anything() && - join.take() == ir::MatchSelection::All; -} - -auto is_streamable_pair_int_join(const ir::JoinNode& join) -> bool { - if (join.kind() != ir::JoinKind::Inner || join.predicate().has_value() || - join.keys().size() != 2 || join.null_match() != ir::NullMatch::Never || - join.expect().asserts_anything() || join.take() != ir::MatchSelection::All) { - return false; - } - const ir::SchemaInfo left_schema = ir::infer_schema(*join.children()[0]); - const ir::SchemaInfo right_schema = ir::infer_schema(*join.children()[1]); - if (!left_schema.is_known() || !right_schema.is_known()) { - return false; - } - // NOLINTNEXTLINE(readability-use-anyofallof) - for (const ir::JoinKey& key : join.keys()) { - const ir::SchemaField* lf = left_schema.find(key.left); - const ir::SchemaField* rf = right_schema.find(key.right); - if (lf == nullptr || rf == nullptr || !lf->type.has_value() || !rf->type.has_value() || - *lf->type != ir::ColumnType::Int64 || *rf->type != ir::ColumnType::Int64) { - return false; - } - } - return true; -} - -auto inner_join_table(const Table& left, const Table& right, const std::vector& keys, - const ir::JoinSuffixPolicy& suffix, - const std::vector& pending_order, const ExecutionContext& exec) - -> std::expected { - // I4 convergence: keep the materialized signature used by interpret_node, - // but run the same hash join that build_operator selects for this exact - // semantic subset. Table sources copy only their column handles. - auto source = make_table_source(left); - return materialize_operator(std::make_unique( - std::move(source), right, &keys, exec, suffix, &pending_order, - resolved_join_parallelism(exec))); -} - -namespace { - -// The single choke point for "build this subtree, then immediately drain it -// to a whole Table" — every call site in this file that needs a materialized -// side (a join's build/probe side, an update's input, an aggregate's fused -// join operand, ...) with no downstream consumer to overlap the build with -// routes through here, rather than each hand-rolling its own -// `build_operator` + `materialize_operator` pair. One place to reason about -// this pattern instead of N independently-drifting copies. -auto materialize_row_local(const ir::Node& node, const TableRegistry& registry, - const ScalarRegistry* scalars, const ExternRegistry* externs, - const ExecutionContext& exec, ModelResult* model_out) - -> std::expected { - auto op = build_operator(node, registry, scalars, externs, exec, model_out); - if (!op.has_value()) { - return std::unexpected(std::move(op.error())); - } - return materialize_operator(std::move(op.value())); -} - -template - -auto build_unary_materializing_operator(const ir::Node& child_node, const TableRegistry& registry, - const ScalarRegistry* scalars, - const ExternRegistry* externs, const ExecutionContext& exec, - ModelResult* model_out, Fn fn) - -> std::expected { - auto materialized = - materialize_row_local(child_node, registry, scalars, externs, exec, model_out); - if (!materialized.has_value()) { - return std::unexpected(std::move(materialized.error())); - } - auto result = fn(std::move(materialized.value())); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - return make_table_source(std::move(result.value())); -} - -} // namespace - -namespace { - -template - -auto build_binary_materializing_operator(const ir::Node& left_node, const ir::Node& right_node, - const TableRegistry& registry, - const ScalarRegistry* scalars, - const ExternRegistry* externs, - const ExecutionContext& exec, ModelResult* model_out, - Fn fn) -> std::expected { - // Multiple producers (plans/parallelism-overview.md): tried overlapping - // `left_node`/`right_node`'s materializations on a raw std::thread here, - // twice. First attempt (unbudgeted): net regression on the full PDS-H - // suite (q09 +57%, q04/q06/q07/q08/q19 also worse). Second attempt, - // under a since-removed helper-thread budget: q09 STILL regressed - // (+47.5%), and the budget provably wasn't the reason -- q09 hits this - // function exactly once (verified with a temporary entry-count trace), - // so there was never a recursive pile-up here for a budget to bound in - // the first place, and re-testing at budget=1/2/8 all gave the same - // ~230-250ms. The actual cost is structural, not concurrency-count: this - // function materializes BOTH sides fully (unlike is_streamable_inner_join, - // which builds the left as a cheap lazy operator and only materializes - // the right), so overlapping two already-expensive full materializations - // contends for the same cores/memory bandwidth rather than filling idle - // ones. Reverted a second time; a future attempt here needs a - // cost-aware gate (e.g. skip when both sides are large), not a - // thread-count budget. - auto left = materialize_row_local(left_node, registry, scalars, externs, exec, model_out); - if (!left.has_value()) { - return std::unexpected(std::move(left.error())); - } - auto right = materialize_row_local(right_node, registry, scalars, externs, exec, model_out); - if (!right.has_value()) { - return std::unexpected(std::move(right.error())); - } - auto result = fn(std::move(left.value()), std::move(right.value())); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - return make_table_source(std::move(result.value())); -} - -auto eval_extern_args(const std::vector& exprs, const ScalarRegistry* scalars, - const ExternRegistry* externs) -> std::expected { - ExternArgs args; - args.reserve(exprs.size()); - for (const auto& arg : exprs) { - auto val = eval_expr(arg, Table{}, 0, scalars, externs); - if (!val.has_value()) { - return std::unexpected(std::move(val.error())); - } - // Externs take null-free ScalarValue arguments (see the null-arm plan). - auto scalar = scalar_from_expr(val.value()); - if (!scalar.has_value()) { - return std::unexpected("null argument in extern function call"); - } - args.push_back(std::move(*scalar)); - } - return args; -} - -} // namespace - -auto invoke_extern_call(const ir::ExternCallNode& ec, const ScalarRegistry* scalars, - const ExternRegistry* externs) -> std::expected { - if (externs == nullptr) { - return std::unexpected("extern call with no registry: " + ec.callee()); - } - const auto* fn = externs->find(ec.callee()); - if (fn == nullptr) { - return std::unexpected("unknown extern function: " + ec.callee()); - } - if (fn->first_arg_is_table) { - return std::unexpected("extern function requires a table input: " + ec.callee()); - } - auto args = eval_extern_args(ec.args(), scalars, externs); - if (!args.has_value()) { - return std::unexpected(std::move(args.error())); - } - if (fn->kind == ExternReturnKind::Table && fn->chunked_table_func) { - auto source = fn->chunked_table_func(args.value()); - if (source.has_value()) { - auto materialized = materialize_operator(std::move(source.value())); - if (!materialized.has_value()) { - return std::unexpected(std::move(materialized.error())); - } - return ExternValue{std::move(materialized.value())}; - } - } - auto result = fn->func(args.value()); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - return result; -} - -auto execute_program_preamble(const std::vector& preamble, - const ScalarRegistry* scalars, const ExternRegistry* externs) - -> std::expected { - for (const auto& node : preamble) { - if (node->kind() != ir::NodeKind::ExternCall) { - return std::unexpected("program preamble only supports extern calls"); - } - const auto& ec = ir::node_cast(*node); - auto result = invoke_extern_call(ec, scalars, externs); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - } - return {}; -} - -/// Planner seam: returns a pull-based operator that, when drained, -/// produces the logical result of `node`. Chunked operators exist -/// today for node kinds that are safe and useful to stream; any other -/// node kind falls back to the full-table `interpret_node` path and -/// is wrapped in a `TableSourceOperator` so downstream chunked -/// operators see a uniform pull-based interface. -// Order-delay past Filter/Project/Rename, and Head/Tail pushdown past -// Project/Rename, are handled by the IR canonicalize pass -// (src/ir/canonicalize.cpp). IR arrives here in canonical form, so -// build_operator only needs one branch per NodeKind and the shapes it -// matches are the post-canonicalization shapes (e.g. Project(Filter(x)) -// for the fused operator, not Project(Filter(Order(x)))). - -namespace { - -// Runtime-multithreading Phase 1, serial morsel slice. Owns the materialized -// input `Table` that the pipeline's `PartitionedTableSource` reads by pointer. -// `input_` is declared before `chain_` so the chain — which holds a raw -// pointer into `input_` — is destroyed first. -class OwningMorselPipelineOperator final : public Operator { - public: - OwningMorselPipelineOperator(std::unique_ptr
input, OperatorPtr chain) - : input_(std::move(input)), chain_(std::move(chain)) {} - - [[nodiscard]] auto next() -> std::expected, std::string> override { - return chain_->next(); - } - - private: - std::unique_ptr
input_; - OperatorPtr chain_; -}; - -/// Serial-slice stand-in for the Phase 1 ordered merger. The current source -/// emits in sequence order, so validating the stream is enough to make a lost, -/// duplicated, or provenance-stripped morsel an immediate error. A later -/// concurrent merger replaces this with sequence-indexed buffering/release. -class SerialMorselOrderValidator final : public Operator { - public: - SerialMorselOrderValidator(OperatorPtr child, std::uint64_t expected_morsels, std::size_t grain) - : child_(std::move(child)), - expected_morsels_(expected_morsels), - grain_(grain == 0 ? 1 : grain) {} - - [[nodiscard]] auto next() -> std::expected, std::string> override { - auto result = child_->next(); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - if (result->has_value()) { - const auto& chunk = result->value(); - const auto expected_offset = static_cast(next_sequence_) * grain_; - if (chunk.sequence != next_sequence_ || chunk.row_offset != expected_offset) { - return std::unexpected("morsel pipeline: morsel identity gap or reordering"); - } - ++next_sequence_; - return result; - } - if (next_sequence_ != expected_morsels_) { - return std::unexpected("morsel pipeline: missing output morsel"); - } - return result; - } - - private: - OperatorPtr child_; - std::uint64_t expected_morsels_ = 0; - std::uint64_t next_sequence_ = 0; - std::size_t grain_ = 1; -}; - -} // namespace - -auto morsel_grain(const ExecutionContext& exec, std::size_t rows) -> std::size_t { - if (exec.parallel_grain != 0) { - return exec.parallel_grain; // explicit override, used as given - } - // Aim for this many morsels per worker, so one slow morsel cannot strand - // the others. Below ~2 per thread the sweep shows real imbalance loss. - constexpr std::size_t kMorselsPerThread = 4; - // The measured good band was 16k-256k; clamp inside it. The upper bound is - // what stops the formula from choosing a grain worse than the old constant - // on a large input — see the declaration. - constexpr std::size_t kMinGrain = 4096; - constexpr std::size_t kMaxGrain = 65536; - - const std::size_t pool_size = process_worker_pool().size(); - const std::size_t budget = exec.compute_budget(); - const std::size_t threads = std::max(std::min(budget, pool_size), 1); - return std::clamp(rows / (threads * kMorselsPerThread), kMinGrain, kMaxGrain); -} - -auto process_pipeline_stats() -> ParallelPipelineStats* { - // File-local, like the worker pool and the query lease: a bundled plugin - // statically links runtime code, so an inline header variable would give - // each plugin its own counter (the RTLD_LOCAL trap). - // - // The reporter is a separate static whose destructor runs at exit. It holds - // no reference to the counters it prints beyond the function-local statics - // above it, which outlive it by declaration order. - static const bool enabled = std::getenv("IBEX_PARALLEL_STATS") != nullptr; - static ParallelPipelineStats stats; - // Function-local exit reporter, instantiated once below; never copied or moved. - // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions) - struct Reporter { - ~Reporter() { - if (!enabled) { - return; - } - ibex::formatting::print( - stderr, - "pipeline stats: parallel={} serial={} morsels={} " - "pipelined_scans={} pipelined_stages={} range_heads={} two_phase={} " - "parallel_fields={} parallel_direct_numeric_fields={} parallel_probes={} " - "parallel_hash_builds={} parallel_aggregate_partitions={} " - "parallel_aggregate_finalizes={} " - "grouped_lifted_group_state={} chunk_direct_updates={}\n", - stats.parallel_pipelines.load(), stats.serial_pipelines.load(), - stats.morsels.load(), stats.pipelined_scans.load(), stats.pipelined_stages.load(), - stats.range_heads.load(), stats.two_phase_filters.load(), - stats.parallel_fields.load(), stats.parallel_direct_numeric_fields.load(), - stats.parallel_probes.load(), stats.parallel_hash_builds.load(), - stats.parallel_aggregate_partitions.load(), - stats.parallel_aggregate_finalizes.load(), stats.grouped_lifted_group_state.load(), - stats.chunk_direct_updates.load()); - } - }; - static const Reporter reporter; - return enabled ? &stats : nullptr; -} - -void configure_parallel_from_env(ExecutionContext& exec) { - // The other two execution switches, applied the same way and for the same - // reason: one authority per setting. Both were `getenv` at their use sites - // until the seams that share them started to outnumber the settings — - // `stream_scans` is read at three build seams that must agree, and - // `parallel_join_probe` at three probe gates. An unset variable leaves the - // caller's choice alone, so a context built by hand still means "ignore the - // environment". - if (const auto want = stream_scans_from_env(); want.has_value()) { - exec.stream_scans = *want; - } - if (const auto want = parallel_join_probe_from_env(); want.has_value()) { - exec.parallel_join_probe = *want; - } - // Pin the COMPUTE budget to the core count. Every compute gate sizes itself - // from `parallel_threads`, falling back to the pool size when it is 0 — and - // the pool is now sized for decode, which wants more threads than cores - // (`decode_oversubscribe`). Without this, growing the pool would - // oversubscribe the compute paths too, and those measurably do not want it - // (q01 +4.6%, q17 +3.2%). With a multiplier of 1 this is the same number the - // fallback produced, so the default configuration is unchanged. - if (exec.parallel_threads == 0) { - exec.parallel_threads = compute_thread_count(); - } - if (exec.parallel_stats == nullptr) { - exec.parallel_stats = process_pipeline_stats(); - } - if (exec.execution_profile == nullptr && execution_profile_requested()) { - // The budget occupancy is measured against. Read from the context or - // the environment rather than `process_worker_pool().size()`, so that - // asking for a profile never constructs a pool a serial query would - // otherwise never have built. - const std::size_t budget = exec.compute_budget(); - exec.execution_profile = std::make_shared(budget); - } - if (const std::size_t grain = morsel_rows_from_env(); grain > 0) { - exec.parallel_grain = grain; - // An explicit grain is an explicit request to partition at that size, - // so it also lowers the serial threshold — otherwise the default - // threshold would silently override the knob it was asked to honor. - exec.parallel_min_rows = std::min(exec.parallel_min_rows, grain); - } - // `parallel_threads` is set at the top of this function, not here: it is - // the COMPUTE budget and must track the core count, because the pool it - // used to default to is now sized for decode instead. -} - -namespace { - -// One construction point for every row-local map operator that can live in a -// morsel pipeline. The serial planner uses the same factory: only the pipeline -// asks maps to retain zero-row morsels, because its ordered merger needs one -// output identity for every input morsel. Keeping the construction (especially -// FUP's gather set) here prevents the two planners from drifting as range-aware -// kernels replace these chunked implementations. -auto physical_filter_route(const ir::Expr& predicate, - const std::vector* source_signature) - -> kernel::FilterChunkRoute { - // A populated signature is the physical planner's proof that this is a - // registered table scan with one of the representations the Chunk gather - // family owns. Lazy/extern sources intentionally arrive without that - // proof and retain the compatibility route. - if (source_signature == nullptr || source_signature->empty() || - !kernel::supports_native_chunk_predicate(predicate)) { - return kernel::FilterChunkRoute::Auto; - } - for (const auto& signature : *source_signature) { - switch (signature.representation) { - case ColumnRepresentation::FixedWidth: - case ColumnRepresentation::PackedBool: - case ColumnRepresentation::StringSlabs: - case ColumnRepresentation::CategoricalCodes: - break; - default: - return kernel::FilterChunkRoute::Auto; - } - switch (signature.null_policy) { - case KernelNullPolicy::AllValid: - case KernelNullPolicy::Nullable: - break; - default: - return kernel::FilterChunkRoute::Auto; - } - } - return kernel::FilterChunkRoute::NativePredicate; -} - -auto build_filter_gather_map(const MapStep& step, OperatorPtr child, const ScalarRegistry* scalars, - const ExternRegistry* /*unused*/, const ExecutionContext& /*unused*/, - const std::vector* source_signature, - bool preserve_empty_morsels) - -> std::expected { - const auto& filter = ir::node_cast(*step.node); - return std::make_unique( - std::move(child), &filter.predicate(), scalars, - physical_filter_route(filter.predicate(), source_signature), preserve_empty_morsels); -} - -auto build_metadata_map(const MapStep& step, OperatorPtr child, const ScalarRegistry* /*unused*/, - const ExternRegistry* /*unused*/, const ExecutionContext& /*unused*/, - const std::vector* /*unused*/, bool /*unused*/) - -> std::expected { - if (step.node->kind() == ir::NodeKind::Project) { - const auto& project = ir::node_cast(*step.node); - return std::make_unique(std::move(child), &project.columns()); - } - const auto& rename = ir::node_cast(*step.node); - return std::make_unique(std::move(child), &rename.renames()); -} - -auto build_row_local_update_map(const MapStep& step, OperatorPtr child, - const ScalarRegistry* scalars, const ExternRegistry* externs, - const ExecutionContext& exec, - const std::vector* /*unused*/, - bool /*unused*/) -> std::expected { - const auto& update = ir::node_cast(*step.node); - return std::make_unique(std::move(child), &update.fields(), scalars, - externs, exec); -} - -auto build_filter_project_gather_map(const MapStep& step, OperatorPtr child, - const ScalarRegistry* scalars, - const ExternRegistry* /*unused*/, - const ExecutionContext& /*unused*/, - const std::vector* source_signature, - bool preserve_empty_morsels) - -> std::expected { - if (step.filter_predicate == nullptr || step.project_columns == nullptr) { - return std::unexpected("filter-project map step missing normalized operands"); - } - return std::make_unique( - std::move(child), step.filter_predicate, step.project_columns, scalars, - physical_filter_route(*step.filter_predicate, source_signature), preserve_empty_morsels); -} - -auto build_filter_update_project_gather_map( - const MapStep& step, OperatorPtr child, const ScalarRegistry* scalars, - const ExternRegistry* externs, const ExecutionContext& exec, - const std::vector* source_signature, bool preserve_empty_morsels) - -> std::expected { - if (step.filter_predicate == nullptr || step.update_fields == nullptr || - step.project_columns == nullptr) { - return std::unexpected("filter-update-project map step missing normalized operands"); - } - robin_hood::unordered_set update_outputs; - robin_hood::unordered_set needed; - for (const auto& field : *step.update_fields) { - update_outputs.insert(field.alias); - collect_expr_column_refs(field.expr, needed); - } - for (const auto& column : *step.project_columns) { - if (!update_outputs.contains(column.name)) { - needed.insert(column.name); - } - } - std::vector gather_columns; - gather_columns.reserve(needed.size()); - for (const auto& name : needed) { - gather_columns.push_back(ir::ColumnRef{.name = name}); - } - return std::make_unique( - std::move(child), step.filter_predicate, step.update_fields, step.project_columns, - std::move(gather_columns), scalars, externs, exec, - physical_filter_route(*step.filter_predicate, source_signature), preserve_empty_morsels); -} - -} // namespace - -auto map_kernel_factory(MapKernelCapability capability) noexcept -> MapKernelFactory { - static constexpr std::array factories{ - &build_filter_gather_map, - &build_metadata_map, - &build_row_local_update_map, - &build_filter_project_gather_map, - &build_filter_update_project_gather_map, - }; - const auto index = static_cast(capability); - return index < factories.size() ? factories[index] : nullptr; -} - -namespace { - -auto build_row_local_map_operator(const MapStep& step, OperatorPtr child, - const ScalarRegistry* scalars, const ExternRegistry* externs, - const ExecutionContext& exec, bool preserve_empty_morsels) - -> std::expected { - const MapKernelFactory factory = - step.factory != nullptr ? step.factory : map_kernel_factory(step.capability); - if (factory == nullptr) { - return std::unexpected("row-local map factory: unknown kernel capability"); - } - return factory(step, std::move(child), scalars, externs, exec, nullptr, preserve_empty_morsels); -} - -/// One unfused node as a step. The compatibility entry point for callers -/// outside a physical plan, which have a node and no fusion to express. -auto build_row_local_map_operator(const ir::Node& node, OperatorPtr child, - const ScalarRegistry* scalars, const ExternRegistry* externs, - const ExecutionContext& exec, bool preserve_empty_morsels) - -> std::expected { - const auto capability = map_kernel_capability(node); - if (!capability.has_value()) { - return std::unexpected("row-local map factory: unsupported kernel capability"); - } - const MapStep step{ - .node = &node, .capability = *capability, .factory = map_kernel_factory(*capability)}; - return build_row_local_map_operator(step, std::move(child), scalars, externs, exec, - preserve_empty_morsels); -} - -// The base of one worker's worker chain: a source the worker points at the -// morsel it just claimed. Two implementations, differing only in whether the -// morsel's rows are copied out of the shared input before the chain sees them. -class MorselSource : public Operator { - public: - /// Aim the source at rows [begin, end) of the pipeline's input. The next - /// `next()` produces exactly that morsel and then reports exhaustion, so - /// one call feeds one turn of the worker loop. - virtual void set_morsel(std::size_t begin, std::size_t end, std::uint64_t sequence) = 0; -}; - -// Gathering source: materializes the morsel, then the chain above runs over it -// exactly as the serial path does. The fallback for any pipeline whose head this -// file cannot evaluate by range. -class GatherMorselSource final : public MorselSource { - public: - explicit GatherMorselSource(const Table& input) : input_(&input) {} - - void set_morsel(std::size_t begin, std::size_t end, std::uint64_t sequence) override { - pending_ = make_morsel_chunk(*input_, begin, end, sequence); - } - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (!pending_.has_value()) { - return std::optional{}; - } - auto chunk = std::move(*pending_); - pending_.reset(); - return std::optional{std::move(chunk)}; - } - - private: - const Table* input_; - std::optional pending_; -}; - -// Range-filtering source: absorbs the pipeline's head `Filter` and evaluates its -// predicate directly over the input's rows [begin, end), so the morsel is never -// materialized. Only surviving rows are ever copied — the gather the serial -// path pays for every row is gone. -// -// This is one operator doing the work of two, so it owes both their contracts: -// the morsel identity a gathering source would have stamped (`sequence` and -// `row_offset`, which the worker loop re-checks), and the head filter's -// `preserve_empty_morsels` behaviour — an empty result is still emitted, -// because the merger indexes by sequence and a skipped morsel is a lost slot -// rather than a smaller answer. -class RangeFilterMorselSource final : public MorselSource { - public: - RangeFilterMorselSource(const Table& input, const ir::Expr* predicate, - const std::vector* project, - const ScalarRegistry* scalars) - : input_(&input), predicate_(predicate), project_(project), scalars_(scalars) {} - - void set_morsel(std::size_t begin, std::size_t end, std::uint64_t sequence) override { - pending_ = ChunkIdentity{.sequence = sequence, .row_offset = begin}; - begin_ = begin; - end_ = end; - } - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (!pending_.has_value()) { - return std::optional{}; - } - const auto identity = *pending_; - pending_.reset(); - const RowRange rows{.begin = begin_, .count = end_ - begin_}; - auto filtered = - project_ == nullptr - ? filter_table_range(*input_, *predicate_, rows, scalars_) - : filter_project_table_range(*input_, *predicate_, *project_, rows, scalars_); - if (!filtered.has_value()) { - return std::unexpected(std::move(filtered.error())); - } - return std::optional{table_to_chunk(std::move(filtered.value()), identity)}; - } - - private: - const Table* input_; - const ir::Expr* predicate_; - const std::vector* project_; - const ScalarRegistry* scalars_; - std::optional pending_; - std::size_t begin_ = 0; - std::size_t end_ = 0; -}; - -/// How a head operator is evaluated by range, when it can be. A filter head is -/// where gathering costs most — it copies rows the predicate is about to throw -/// away — and the physically fused filter/project form is included as well. -struct RangeHead { - const ir::Expr* predicate = nullptr; - const std::vector* project = nullptr; ///< null when unfused -}; - -/// The head's range form, or nullopt when it has to be built above a gathered -/// morsel instead. -/// -/// Two things disqualify a head: -/// -/// - A predicate that is not `is_range_native_expr`. Pipeline eligibility admits -/// Scalar calls, but every call still evaluates whole-table-and-slice, so -/// absorbing `abs(a) > 50` would re-run `abs` over the entire input once per -/// morsel — measured at 10x slower than serial on 20M rows. Gathering is the -/// correct choice there: the morsel is materialized once and the predicate -/// then runs over morsel-sized data. -/// - A column-less table, whose row count lives in the chunk's `logical_rows` -/// rather than in any column; only the gathering source carries that over. -[[nodiscard]] auto range_filter_head(const MapStep& step, const Table& input) - -> std::optional { - if (input.columns.empty()) { - return std::nullopt; - } - if (step.capability == MapKernelCapability::FilterGather) { - // A head absorbs a filter and, at most, the projection directly above - // it. A fused step that also carries an Update computes columns between - // the two, and absorbing the projection here would skip that - // computation -- the projection would then name a column nothing - // produced. Such a step keeps its operator chain. - if (step.fused_update != nullptr) { - return std::nullopt; - } - const auto& predicate = ir::node_cast(*step.node).predicate(); - if (!is_range_native_expr(predicate)) { - return std::nullopt; - } - // A planner-fused Project rides along exactly as the fused IR kind's - // column list does below: same head, same absorbed projection. - const std::vector* project = - step.fused_project != nullptr - ? &ir::node_cast(*step.fused_project).columns() - : nullptr; - return RangeHead{.predicate = &predicate, .project = project}; - } - if (step.capability == MapKernelCapability::FilterProjectGather && - step.update_fields == nullptr && step.filter_predicate != nullptr && - step.project_columns != nullptr) { - if (!is_range_native_expr(*step.filter_predicate)) { - return std::nullopt; - } - return RangeHead{.predicate = step.filter_predicate, .project = step.project_columns}; - } - return std::nullopt; -} - -// One worker's private copy of the pipeline's map chain. The operators are -// per-worker (they carry mutable per-chunk state); the IR nodes, registries, -// and the input table they read are shared and immutable for the pipeline's -// lifetime. -struct MorselWorkerChain { - MorselSource* source = nullptr; // owned by `chain`, re-aimed per morsel - OperatorPtr chain; -}; - -[[nodiscard]] auto build_morsel_worker_chain(const std::vector& operators, - const Table& input, const ScalarRegistry* scalars, - const ExternRegistry* externs, - const ExecutionContext& exec, - const JoinProbe* probe_head = nullptr) - -> std::expected { - // A qualifying head is absorbed into the source rather than built as an - // operator above it — same output, without materializing the morsel first. - // Not available under a probe head: that optimization reads the first - // operator as a filter over the SOURCE's rows, and with a probe between - // them the source's rows are the probe side, not the filter's input. - std::size_t first_op = 0; - std::unique_ptr source; - if (!operators.empty() && probe_head == nullptr) { - if (auto head = range_filter_head(operators.front(), input); head.has_value()) { - source = std::make_unique(input, head->predicate, - head->project, scalars); - first_op = 1; - } - } - if (source == nullptr) { - source = std::make_unique(input); - } - - MorselWorkerChain worker{.source = source.get(), .chain = std::move(source)}; - // The probe runs first, on this worker's own morsel of the probe side, and - // every map above it runs in the same worker on the probe's output. That - // is the whole point of the fused shape: the join's output is never - // assembled as a table between the probe and the maps. - if (probe_head != nullptr) { - worker.chain = std::make_unique(std::move(worker.chain), *probe_head, - /*preserve_empty_morsels=*/true); - } - for (std::size_t i = first_op; i < operators.size(); ++i) { - const MapStep& op_node = operators[i]; - // `preserve_empty_morsels` is what makes one input morsel yield exactly - // one identified output morsel — the merger indexes by sequence, so a - // silently coalesced empty result would be a lost slot, not a smaller - // answer. - auto next = build_row_local_map_operator(op_node, std::move(worker.chain), scalars, externs, - exec, true); - if (!next.has_value()) { - // The plan's step vocabulary only admits row-local map kinds. - return std::unexpected("morsel pipeline: " + next.error()); - } - worker.chain = std::move(next.value()); - } - return worker; -} -/// A bounded, sequence-ordered handoff between several producers and one -/// consumer — the one implementation of that shape in the runtime. -/// -/// Slots are addressed `sequence % window`, so a producer may run at most -/// `window` sequences ahead of the consumer and then parks; the consumer parks -/// on the slot it needs next. Both waits are wrapped in `RingWaitScope`, -/// because produced-ahead and waiting-on-workers are idle rather than work — -/// counting them as work makes a blocked worker read as a busy one and -/// overstates occupancy. -/// -/// Failure is ordered by sequence, not by arrival: the lowest-sequence failure -/// is the one reported, so the message a query returns never depends on which -/// thread lost a race. `record_fault` takes a static string and allocates -/// nothing, which is the only reporting path still available when allocation is -/// what failed. -/// -/// Producer liveness is tracked so the consumer cannot wait for a sequence that -/// is never coming: a producer that leaves for any reason — exhaustion, error, -/// exception — must call `producer_exited`, which is what turns "a worker died" -/// into an error rather than a hang. -class OrderedChunkRing { - public: - OrderedChunkRing(std::size_t window, std::size_t producers) - : window_(window == 0 ? 1 : window), - ring_(window == 0 ? 1 : window), - active_producers_(producers) {} - - OrderedChunkRing(const OrderedChunkRing&) = delete; - auto operator=(const OrderedChunkRing&) -> OrderedChunkRing& = delete; - OrderedChunkRing(OrderedChunkRing&&) = delete; - auto operator=(OrderedChunkRing&&) -> OrderedChunkRing& = delete; - ~OrderedChunkRing() = default; - - /// What a producer should do with the sequence it just claimed. - enum class Acquire : std::uint8_t { - Proceed, ///< the slot is free; produce into it - Abandon, ///< cancelled, or a lower sequence already failed - }; - - /// Park until this sequence's slot is free. Called with no lock held. - [[nodiscard]] auto acquire(std::uint64_t sequence) -> Acquire { - std::unique_lock lock(mutex_); - const RingWaitScope ring_wait; - space_.wait(lock, [&] { - return cancelled_ || sequence < released_ + window_ || - (has_error_ && error_sequence_ < sequence); - }); - // Only ever abandons sequences above the reported failure, so the - // consumer still receives everything below it. - return (cancelled_ || (has_error_ && error_sequence_ < sequence)) ? Acquire::Abandon - : Acquire::Proceed; - } - - void publish(std::uint64_t sequence, Chunk chunk) { - { - const std::scoped_lock lock(mutex_); - ring_[static_cast(sequence % window_)] = std::move(chunk); - } - ready_.notify_one(); - } - - /// Record an owned message. The caller has already built the string, so - /// taking it by value and moving it under the lock never allocates here. - void record_error(std::uint64_t sequence, std::string message) noexcept { - { - const std::scoped_lock lock(mutex_); - if (claim_failure(sequence)) { - error_owned_ = std::move(message); - error_fixed_ = nullptr; - } - } - wake_all(); - } - - /// Record a message in static storage. Allocates nothing at all, so it is - /// the only reporting path available once allocation is what failed. - void record_fault(std::uint64_t sequence, const char* message) noexcept { - { - const std::scoped_lock lock(mutex_); - if (claim_failure(sequence)) { - error_owned_.clear(); // frees, never allocates - error_fixed_ = message; - } - } - wake_all(); - } - - void producer_exited() noexcept { - { - const std::scoped_lock lock(mutex_); - --active_producers_; - } - ready_.notify_all(); - } - - /// Take the chunk at `sequence`, or nullopt when the run stopped before - /// producing it — cancelled, failed, or out of producers. The caller asks - /// `failure()` for why. - [[nodiscard]] auto take(std::uint64_t sequence) -> std::optional { - std::optional chunk; - { - std::unique_lock lock(mutex_); - const auto slot = static_cast(sequence % window_); - { - const RingWaitScope ring_wait; - ready_.wait(lock, [&] { - return ring_[slot].has_value() || cancelled_ || active_producers_ == 0 || - (has_error_ && error_sequence_ <= sequence); - }); - } - if (ring_[slot].has_value()) { - chunk = std::move(ring_[slot]); - ring_[slot].reset(); - ++released_; - } - } - if (chunk.has_value()) { - space_.notify_all(); - } - return chunk; - } - - [[nodiscard]] auto failure() const -> std::optional { - const std::scoped_lock lock(mutex_); - if (!has_error_) { - return std::nullopt; - } - return error_fixed_ != nullptr ? std::string(error_fixed_) : error_owned_; - } - - void cancel() noexcept { - { - const std::scoped_lock lock(mutex_); - cancelled_ = true; - } - wake_all(); - } - - private: - /// True if `sequence` becomes the reported failure. Lowest sequence wins, - /// so the error a query reports never depends on thread timing. - [[nodiscard]] auto claim_failure(std::uint64_t sequence) noexcept -> bool { - if (has_error_ && sequence >= error_sequence_) { - return false; - } - has_error_ = true; - error_sequence_ = sequence; - return true; - } - - void wake_all() noexcept { - ready_.notify_all(); - space_.notify_all(); - } - - std::size_t window_; - mutable std::mutex mutex_; - std::condition_variable ready_; // consumer waits for the next sequence - std::condition_variable space_; // producers wait for ring space - std::vector> ring_; - std::uint64_t released_ = 0; - std::size_t active_producers_ = 0; - bool cancelled_ = false; - // The failure channel is split so it can be written without allocating. - // `error_owned_` carries a message moved in from a producer; `error_fixed_` - // points at static storage. - bool has_error_ = false; - std::uint64_t error_sequence_ = 0; - std::string error_owned_; - const char* error_fixed_ = nullptr; -}; - -// Runtime-multithreading Phase 1: the morsel pipeline executor. -// -// Workers pull numbered morsels from one shared cursor over the immutable -// materialized input, run their own chain over each, and deposit the result in -// a bounded ring indexed by `sequence`. `next()` is the ordered merger: it -// releases results strictly in sequence order, so the operator's output is -// byte-identical to the serial chain's regardless of completion order. The ring -// is the plan's bounded in-flight queue — a worker that runs ahead of the -// consumer by a full window blocks instead of buffering the whole pipeline. -// -// Output ownership (the plan's Phase-1 allocator variable): each task owns the -// chunk it produces, and the merger's consumer moves it straight into the -// downstream `MaterializeOperator` concat. Nothing escapes into task-local -// scratch storage, so no arena ownership has to be transferred. That is the -// simplest of the strategies the plan allows and the one whose allocation -// behavior the acceptance benchmarks measure; a presized query-owned buffer -// pool is the next option if allocation shows up in those numbers. -// -// Error and cancellation determinism: a failing morsel records its error under -// the lock, keeping the *lowest* sequence, and workers abandon only morsels -// above it — so every morsel below the reported failure is still produced, and -// the error a query reports does not depend on thread timing. -class MorselPipelineOperator final : public Operator { - public: - MorselPipelineOperator(std::unique_ptr
input, std::vector workers, - std::size_t grain, std::uint64_t morsel_count, WorkerPool& pool) - : input_(std::move(input)), - workers_(std::move(workers)), - grain_(grain == 0 ? 1 : grain), - morsel_count_(morsel_count), - pool_(&pool), - ring_(std::max(workers_.size() * 2, 2), workers_.size()) {} - - ~MorselPipelineOperator() override { cancel_and_join(); } - - MorselPipelineOperator(const MorselPipelineOperator&) = delete; - auto operator=(const MorselPipelineOperator&) -> MorselPipelineOperator& = delete; - MorselPipelineOperator(MorselPipelineOperator&&) = delete; - auto operator=(MorselPipelineOperator&&) -> MorselPipelineOperator& = delete; - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (finished_) { - return std::optional{}; - } - if (!started_) { - started_ = true; - batch_ = pool_->submit(workers_.size(), [this](std::size_t id) { run_worker(id); }); - } - if (next_sequence_ >= morsel_count_) { - return finish(); - } - if (interrupt_requested()) { - return fail(interrupt_message()); - } - - std::optional chunk = ring_.take(next_sequence_); - if (chunk.has_value()) { - ++next_sequence_; - return std::optional{std::move(*chunk)}; - } - - // No chunk: the pipeline stopped early. Report why, deterministically. - // - // An interrupt outranks a recorded data error. A worker that fails at - // the moment the user hits Ctrl+C is a race, and reporting its message - // would make cancellation surface as an arbitrary query error depending - // on which thread won. The cancellation contract says such a query - // reports "interrupted", so the interrupt is checked first. - if (interrupt_requested()) { - return fail(interrupt_message()); - } - if (auto failure = ring_.failure(); failure.has_value()) { - return fail(std::move(*failure)); - } - return fail("morsel pipeline: missing output morsel"); - } - - private: - // Whatever happens to a worker — normal exhaustion, a recorded error, or an - // exception — it must stop counting as active and must wake the merger. - // Skipping this on any path leaves the consumer waiting for a sequence that - // is never coming, which is a hang rather than an error. - void worker_exited() noexcept { ring_.producer_exited(); } - - void run_worker(std::size_t worker_id) noexcept { - // Cleanup runs however this scope is left, so no path can leave the - // consumer waiting on a worker that is gone. - // One-shot scope guard: aggregate-initialised, never copied or moved. - // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions) - struct ExitGuard { - MorselPipelineOperator* self; - ~ExitGuard() { self->worker_exited(); } - } const guard{this}; - - std::uint64_t sequence = 0; - try { - run_worker_loop(worker_id, sequence); - } catch (const std::exception& error) { - // An exception is not part of the operator protocol (evaluation - // reports failure through `expected`), so it is something - // unplanned — an allocation failure while materializing a morsel, - // say. Convert it to a sequence-tagged pipeline error so it obeys the - // same lowest-sequence determinism as any other failure, rather - // than unwinding through a pool thread. - // - // Composing that message allocates, and the exception this handler - // most expects is `bad_alloc` — so the detailed message is - // best-effort, with an allocation-free fallback underneath it. - // Throwing from here would terminate the process, since this - // function is noexcept precisely so a worker cannot unwind into the - // pool. `what()` cannot be stored: it dies with the exception. - try { - ring_.record_error( - sequence, "morsel pipeline: worker exception: " + std::string(error.what())); - } catch (...) { - ring_.record_fault(sequence, - "morsel pipeline: worker exception (no memory to report it)"); - } - } catch (...) { - ring_.record_fault(sequence, "morsel pipeline: worker threw a non-standard exception"); - } - } - - void run_worker_loop(std::size_t worker_id, std::uint64_t& claimed) { - auto& worker = workers_[worker_id]; - const std::size_t rows = input_->rows(); - while (true) { - const std::uint64_t sequence = cursor_.fetch_add(1, std::memory_order_relaxed); - if (sequence >= morsel_count_) { - break; - } - // Published so an exception thrown below is attributed to the - // morsel that was in flight, not to sequence 0. - claimed = sequence; - // Backpressure: this morsel's slot is only free once the consumer - // has released the morsel `window` ahead of it. - if (ring_.acquire(sequence) == OrderedChunkRing::Acquire::Abandon) { - break; - } - if (interrupt_requested()) { - ring_.cancel(); - break; - } - - const auto [begin, end] = morsel_row_range(rows, grain_, sequence); - worker.source->set_morsel(begin, end, sequence); - auto produced = worker.chain->next(); - - if (!produced.has_value()) { - ring_.record_error(sequence, std::move(produced.error())); - break; - } - if (!produced->has_value()) { - ring_.record_fault(sequence, "morsel pipeline: worker produced no output morsel"); - break; - } - Chunk out = std::move(**produced); - if (out.sequence != sequence || out.row_offset != begin) { - ring_.record_fault(sequence, "morsel pipeline: morsel identity gap or reordering"); - break; - } - ring_.publish(sequence, std::move(out)); - } - } - - // Called from the destructor, so nothing here may throw: an escaping - // exception during destruction terminates the process. Worker bodies are - // already noexcept and convert failures into pipeline errors, so there is - // nothing for `wait()` to rethrow — this guards the path regardless. - void cancel_and_join() noexcept { - try { - ring_.cancel(); - batch_.wait(); - } catch (...) { // NOLINT(bugprone-empty-catch) - // Nothing left to report: the caller is either unwinding or has - // already chosen the message it will return. - } - } - - // Drain the pipeline cleanly at EOF, then check the per-worker chains really - // are exhausted: a chain still holding a suppressed schema carrier would - // mean a morsel was coalesced away rather than emitted. - [[nodiscard]] auto finish() -> std::expected, std::string> { - finished_ = true; - batch_.wait(); - // Same precedence as `next()`: a cancelled run reports cancellation - // even if a worker also failed on its way out. - if (interrupt_requested()) { - return std::unexpected(interrupt_message()); - } - if (auto failure = ring_.failure(); failure.has_value()) { - return std::unexpected(std::move(*failure)); - } - for (auto& worker : workers_) { - auto trailing = worker.chain->next(); - if (!trailing.has_value()) { - return std::unexpected(std::move(trailing.error())); - } - if (trailing->has_value()) { - return std::unexpected("morsel pipeline: unexpected trailing morsel"); - } - } - return std::optional{}; - } - - [[nodiscard]] auto fail(std::string message) - -> std::expected, std::string> { - finished_ = true; - cancel_and_join(); - return std::unexpected(std::move(message)); - } - - // `input_` is declared first so it outlives `workers_`: the chains read it - // through raw pointers, and the batch is joined before any member is - // destroyed. - std::unique_ptr
input_; - std::vector workers_; - std::size_t grain_ = 1; - std::uint64_t morsel_count_ = 0; - WorkerPool* pool_; - - std::atomic cursor_{0}; - - // The ordered handoff between the workers and this operator's `next()`. - OrderedChunkRing ring_; - - std::uint64_t next_sequence_ = 0; - bool started_ = false; - bool finished_ = false; - WorkerPool::Batch batch_; -}; - -// Runtime-multithreading Phase 2: the two-phase parallel filter. -// -// What the ordered merger above cannot remove is the merge itself. Each worker -// materializes its morsel's surviving rows, and `MaterializeOperator` then -// copies all of them again into one table — so a filter pipeline copies its -// output twice where the serial path copies it once. That is why morsel parallelism wins -// track OUTPUT size rather than input size: a selective predicate wins easily, -// and a bulk one loses no matter how much input work is parallelized. -// -// A filter cannot simply presize its output and skip the merge, because its -// cardinality is data-dependent — nobody knows where morsel 7's rows belong -// until morsels 0-6 have been counted. So run the filter in two passes: -// -// Phase A every morsel evaluates the predicate and packs its surviving -// rows into keep words, in parallel. Only the counts matter after. -// (serial) an exclusive prefix sum over those counts gives each morsel the -// row — and, for string columns, the byte — where its output -// begins. The output is then allocated ONCE, at exactly the -// final size. -// Phase B every morsel gathers its rows straight into that shared output -// at its own offset, in parallel. The slices are disjoint, so no -// locking is needed and nothing is copied twice. -// -// The result is emitted as ONE chunk, which `MaterializeOperator` moves instead -// of concatenating. Ordering is structural — a morsel's rows land at its -// prefix-sum offset — so there is no ring, no merger, and the output is -// byte-identical to the serial filter's. -// -// What it costs: phase A's keep words are held for every morsel at once, which -// is one bit per input row (2.5MB for 20M rows), and phase B re-walks them. -// Neither re-evaluates the predicate. -// -// Writing into disjoint output ROWS is only disjoint in MEMORY for columns -// storing at least one addressable unit per row. `Column` and validity -// bitmaps pack 64 rows to a word, so two morsels meeting mid-word touch the -// same word; `gather_selection_into` resolves that with the shared-word rule -// (see `SharedBitWords` in filter.cpp) rather than excluding those columns. -// `filter_gather_is_thread_safe` remains as the allowlist that keeps a future -// column kind out until someone has checked it. -// -// Note a 64-row-aligned grain would NOT have made those columns safe, which is -// the tempting shortcut: an output offset is the prefix sum of POPCOUNTS, not -// of morsel sizes, so a morsel keeping 37 of its 64 rows already leaves the -// next one starting mid-word. Grain only aligns the SOURCE read, and reads -// never race. -class TwoPhaseFilterOperator final : public Operator { - public: - TwoPhaseFilterOperator(std::unique_ptr
input, const ir::Expr& predicate, - bool fused_project, std::vector tail, - const ScalarRegistry* scalars, FilterOutputLayout layout, - std::size_t grain, std::uint64_t morsel_count, std::size_t workers, - WorkerPool& pool) - : input_(std::move(input)), - predicate_(&predicate), - fused_project_(fused_project), - tail_(std::move(tail)), - scalars_(scalars), - layout_(std::move(layout)), - grain_(grain == 0 ? 1 : grain), - morsel_count_(morsel_count), - workers_(workers), - pool_(&pool) { - selections_.resize(static_cast(morsel_count_)); - row_at_.assign(static_cast(morsel_count_), 0); - } - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (done_) { - return std::optional{}; - } - done_ = true; - auto table = run(); - if (!table.has_value()) { - return std::unexpected(std::move(table.error())); - } - // Sequence 0 / row_offset 0: this operator emits the pipeline's whole - // output at once, so it is trivially the first and only morsel. - return std::optional{table_to_chunk(std::move(table.value()), ChunkIdentity{})}; - } - - private: - [[nodiscard]] auto run() -> std::expected { - const std::size_t n_cols = layout_.output.columns.size(); - const bool has_strings = std::ranges::any_of(layout_.src_of_dst, [&](std::size_t src) { - return std::holds_alternative>(*input_->columns[src].column); - }); - if (has_strings) { - chars_at_.assign(static_cast(morsel_count_), - std::vector(n_cols, 0)); - } - - if (auto failure = run_over_morsels([this](std::uint64_t sequence) { phase_a(sequence); }); - failure.has_value()) { - return std::unexpected(std::move(*failure)); - } - - // Exclusive prefix sums: each morsel's counts become its offsets, and - // the running totals become the output's exact size. Serial on purpose - // — it is O(morsels), not O(rows). - std::size_t rows_total = 0; - for (std::size_t m = 0; m < selections_.size(); ++m) { - row_at_[m] = rows_total; - rows_total += selections_[m].kept; - } - std::vector chars_total(n_cols, 0); - if (has_strings) { - for (std::size_t d = 0; d < n_cols; ++d) { - for (auto& per_morsel : chars_at_) { - const std::size_t count = per_morsel[d]; - per_morsel[d] = chars_total[d]; - chars_total[d] += count; - } - } - } - - presize_filter_output(layout_.output, *input_, layout_.src_of_dst, rows_total, chars_total); - - if (auto failure = run_over_morsels([this](std::uint64_t sequence) { phase_b(sequence); }); - failure.has_value()) { - return std::unexpected(std::move(*failure)); - } - - // Identical rule to the serial filter: a row-local filter preserves - // order and time index; a fused projection keeps each only when its - // column survives. - apply_table_properties( - layout_.output, TableProperties::derive( - table_properties_of(*input_), - [&](const std::string& name) -> KeyFate { - return (!fused_project_ || layout_.output.index.contains(name)) - ? KeyFate::kept(name) - : KeyFate::dropped(); - }, - RowTransform::Subset)); - - // Metadata-only operators above the filter run ONCE over the finished - // output rather than per morsel. `project_table` / `rename_table` build - // their result with `add_column_shared` — zero rows copied, O(columns) - // — so running them serially here costs nothing, while routing the - // chain through the ordered merger to parallelize them costs the whole - // merge copy. Measured: `filter … rename` at 93% selectivity was 2.3x - // SLOWER than serial on the merger. - // - // Applying them to the concatenated output is equivalent to applying - // them per morsel because neither reads a row; and because these are - // the same two functions the serial path calls, the ordering and - // time-index rules cannot diverge from it either. - Table result = std::move(layout_.output); - for (const ir::Node* node : tail_) { - auto next = apply_metadata_only(*node, result); - if (!next.has_value()) { - return std::unexpected(std::move(next.error())); - } - result = std::move(next.value()); - } - return result; - } - - [[nodiscard]] static auto apply_metadata_only(const ir::Node& node, const Table& input) - -> std::expected { - switch (node.kind()) { - case ir::NodeKind::Project: - return project_table(input, ir::node_cast(node).columns()); - case ir::NodeKind::Rename: - return rename_table(input, ir::node_cast(node).renames()); - default: - // The pipeline builder only admits `is_metadata_only_node` kinds - // into `tail_`, so reaching this means the two have drifted. - invariant_violation("two-phase filter: non-metadata operator in the tail"); - } - } - - void phase_a(std::uint64_t sequence) { - const auto rows = morsel_range(sequence); - auto selection = - compute_filter_selection(*input_, *predicate_, scalars_, rows, /*row_limit=*/0); - if (!selection.has_value()) { - record_error(sequence, std::move(selection.error())); - return; - } - auto& slot = selections_[static_cast(sequence)]; - slot = std::move(selection.value()); - if (!chars_at_.empty()) { - count_selected_chars(*input_, layout_.src_of_dst, slot, rows, - chars_at_[static_cast(sequence)]); - } - } - - void phase_b(std::uint64_t sequence) { - const auto index = static_cast(sequence); - gather_selection_into( - layout_.output, *input_, layout_.src_of_dst, selections_[index], morsel_range(sequence), - GatherDest{.row = row_at_[index], - .char_base = chars_at_.empty() ? nullptr : &chars_at_[index]}); - } - - [[nodiscard]] auto morsel_range(std::uint64_t sequence) const -> RowRange { - const auto [begin, end] = morsel_row_range(input_->rows(), grain_, sequence); - return RowRange{.begin = begin, .count = end - begin}; - } - - // Run `body` over every morsel across the pool and join. A failure is - // reported with the LOWEST morsel sequence, the same determinism rule the - // ordered merger uses, so which morsel's error a query reports never - // depends on thread timing. Workers abandon only morsels ABOVE a recorded - // failure, so no morsel below it is skipped. - template - [[nodiscard]] auto run_over_morsels(const Body& body) -> std::optional { - reset_failure(); - std::atomic cursor{0}; - { - auto batch = pool_->submit(workers_, [&](std::size_t) noexcept { - while (true) { - const std::uint64_t sequence = cursor.fetch_add(1, std::memory_order_relaxed); - if (sequence >= morsel_count_) { - return; - } - if (failure_below(sequence) || interrupt_requested()) { - return; - } - // A worker may not unwind into the pool, so an unplanned - // exception becomes a sequence-tagged error like any other. - // The message itself allocates and the exception most - // expected here is bad_alloc, so there is an - // allocation-free fallback underneath it. - try { - body(sequence); - } catch (const std::exception& error) { - try { - record_error(sequence, "parallel filter: worker exception: " + - std::string(error.what())); - } catch (...) { - record_fault(sequence, - "parallel filter: worker exception (no memory to " - "report it)"); - } - } catch (...) { - record_fault(sequence, - "parallel filter: worker threw a non-standard exception"); - } - } - }); - batch.wait(); - } - // Same precedence as the ordered merger: an interrupt outranks a - // recorded data error, so a worker failing as Ctrl+C arrives still - // reports cancellation rather than an arbitrary error. - if (interrupt_requested()) { - return interrupt_message(); - } - const std::scoped_lock lock(mutex_); - if (!has_error_) { - return std::nullopt; - } - return error_fixed_ != nullptr ? std::string(error_fixed_) : error_owned_; - } - - void reset_failure() noexcept { - const std::scoped_lock lock(mutex_); - has_error_ = false; - error_fixed_ = nullptr; - error_owned_.clear(); - } - - [[nodiscard]] auto failure_below(std::uint64_t sequence) noexcept -> bool { - const std::scoped_lock lock(mutex_); - return has_error_ && error_sequence_ < sequence; - } - - void record_error(std::uint64_t sequence, std::string message) { - const std::scoped_lock lock(mutex_); - if (claim_failure(sequence)) { - error_owned_ = std::move(message); - error_fixed_ = nullptr; - } - } - - void record_fault(std::uint64_t sequence, const char* message) noexcept { - const std::scoped_lock lock(mutex_); - if (claim_failure(sequence)) { - error_owned_.clear(); // frees, never allocates - error_fixed_ = message; - } - } - - [[nodiscard]] auto claim_failure(std::uint64_t sequence) noexcept -> bool { - if (has_error_ && sequence >= error_sequence_) { - return false; - } - has_error_ = true; - error_sequence_ = sequence; - return true; - } - - // `input_` is declared first so it outlives everything reading it. - std::unique_ptr
input_; - const ir::Expr* predicate_; - bool fused_project_ = false; - /// Metadata-only operators above the filter, source-to-sink, applied once - /// to the finished output. Every element is `is_metadata_only_node`. - std::vector tail_; - const ScalarRegistry* scalars_; - FilterOutputLayout layout_; - std::size_t grain_ = 1; - std::uint64_t morsel_count_ = 0; - std::size_t workers_ = 0; - WorkerPool* pool_; - - // Written by phase A, read by phase B. Every element is touched by exactly - // one worker (indexed by its own morsel sequence), so these need no lock — - // the join between the phases is the synchronization. - std::vector selections_; - std::vector row_at_; - std::vector> chars_at_; // empty when no string column - - std::mutex mutex_; - bool has_error_ = false; - std::string error_owned_; - const char* error_fixed_ = nullptr; - std::uint64_t error_sequence_ = 0; - - bool done_ = false; -}; - -// How many workers a pipeline of `morsel_count` morsels over `rows` rows should -// run on: 0 means "stay on the serial morsel chain". -// -// This is the plan's grain-size serial threshold. Below it, task dispatch, -// ring synchronization, and the merge cost more than the map they parallelize — -// cache-resident work should not pay for threads. A single morsel is serial by -// definition, and a one-thread budget means the caller asked for serial. -/// Whether this input is worth morselizing at all — a *different* question from -/// how many workers it deserves, and conflating the two is a trap worth naming. -/// -/// A "refused" pipeline used to mean a serial sweep of morsels, which still pays -/// per-morsel materialization and the merge concat. So refusing by dropping the -/// worker count made a small query **slower than never forming a pipeline**: -/// measured 100ms against 36ms for the plain serial path, and it got worse once -/// the grain was derived, because that turned 2 morsels into 32. When the -/// answer is no, the input has to run as ONE whole-table chunk. -/// -/// Two thresholds, because a pipeline's cost has two dimensions. Rows alone -/// cannot express it: 131,072 rows won at 6 columns and lost at 2 on the very -/// same predicate, and every row threshold puts those on the same side. -[[nodiscard]] auto is_worth_morselizing(const ExecutionContext& exec, std::size_t rows, - std::size_t columns) -> bool { - if (rows < exec.parallel_min_rows) { - return false; - } - return exec.parallel_min_cells == 0 || columns == 0 || - rows * columns >= exec.parallel_min_cells; -} - -[[nodiscard]] auto morsel_worker_count(const ExecutionContext& exec, std::uint64_t morsel_count) - -> std::size_t { - if (morsel_count < 2 || !exec.can_fan_out()) { - return 0; - } - // Past the parallel gate: consulting the pool here is free of the - // construct-before-declining hazard because a parallel query has already - // built it (or is about to, on its first fan-out). - const std::size_t pool_size = process_worker_pool().size(); - const std::size_t budget = exec.compute_budget(); - const std::size_t workers = - std::min({budget, pool_size, static_cast(morsel_count)}); - return workers < 2 ? 0 : workers; -} - -// Build one eligible row-local parallel-map chain as a morsel pipeline: materialize its -// input subtree once, then run the chain over morsels of that table instead of -// one whole-table chunk. The operators are ordered source-to-sink. -// -// Two executors, one morsel model. A large input fans out across the worker -// pool and is reassembled by `MorselPipelineOperator`'s ordered merger; a small -// one (or a single-threaded budget) runs the same morsels serially through a -// `PartitionedTableSource`, where `MaterializeOperator`'s in-order concat is -// the trivially ordered merger. Both stamp and check the same morsel identity, -// so both are byte-identical to the plain serial chain — which is exactly what -// lets the threshold move without changing an answer. -// -// LOAD-BEARING INVARIANT — materialize before fan-out. The input subtree is -// executed to a `Table` here, on this thread, and every morsel source below -// takes that finished table by reference. That is what makes a deferred/lazy -// source safe in a pipeline: its decode runs exactly once, serially, before any -// worker exists, so neither `LazyTable::cache_` nor a plugin's `decode_` -// closure is ever touched concurrently. It is why `build_operator`'s seam no -// longer screens pipelines for deferred sources. -// -// The morsel sources all take `const Table&`, so the invariant is enforced by -// their signatures rather than by a check. Streaming a source's morsels -// directly into workers would mean handing them something other than a -// finished table — at which point the LazyTable synchronization contract -// applies in full and eligibility has to be re-established. -/// Defined below; both are consulted by the run builder, which decides its own -/// source strategy. -[[nodiscard]] auto scan_pipeline_worker_count(std::size_t unit_count) -> std::size_t; - -/// Defined below; the run builder chooses between this streaming source and -/// materialize-then-morselize, so the choice lives with the run rather than at -/// the construction seam. -[[nodiscard]] auto build_pipelined_scan(const std::vector& operators, - bool count_as_pipeline, const DeferredScan& scan, - std::vector units, +auto build_unary_materializing_operator(const ir::Node& child_node, const TableRegistry& registry, const ScalarRegistry* scalars, - const ExternRegistry* externs, const ExecutionContext& exec) - -> std::expected; - -} // namespace - -// Internal linkage to match the forward declarations beside the join, which -// sit inside this TU's anonymous namespace; the definitions must live down -// here because they use the morsel executor, which is defined below the join. -namespace { - -/// Build a streaming join here and take its probe, so a map pipeline above it -/// can run that probe at the head of its own worker chains. -/// -/// Returns an empty optional -- not an error -- for every join this cannot -/// fuse: a materializing one, a semi/anti one, a deferred probe scan (whose -/// right subtree must be interpreted by the join itself, after it publishes -/// build-side bounds), and any orientation that leaves no materialized probe -/// side. The caller then materializes the join's output as it always did. -/// -/// Narrow on purpose. The point is to establish that two pipelines meeting at -/// a barrier can be built and can produce the right answer; widening the -/// shapes is cheap once that is true, and pointless before. -auto try_take_join_probe(const ir::Node& node, const TableRegistry& registry, - const ScalarRegistry* scalars, const ExternRegistry* externs, - const ExecutionContext& exec, ModelResult* model_out) - -> std::expected, std::string> { - if (std::getenv("IBEX_PROBE_MORSELS") == nullptr) { - return std::optional{}; - } - const physical::Plan plan = physical::plan_physical(node, registry, externs); - const physical::JoinPlan& jp = plan.join; - if (!jp.describes || jp.strategy != physical::JoinStrategy::StreamingProbe || - (jp.branch != physical::JoinBranch::SingleKeyInner && - jp.branch != physical::JoinBranch::PairIntInner)) { - return std::optional{}; - } - const auto& join = ir::node_cast(node); - if (deferred_probe_scan_of(*join.children()[1], exec).scan != nullptr) { - return std::optional{}; - } - - auto left_op = build_operator(*join.children()[0], registry, scalars, externs, exec, model_out); - if (!left_op.has_value()) { - return std::unexpected(std::move(left_op.error())); - } - auto right = - materialize_row_local(*join.children()[1], registry, scalars, externs, exec, model_out); - if (!right.has_value()) { - return std::unexpected(std::move(right.error())); - } - auto op = std::make_unique( - std::move(left_op.value()), std::move(right.value()), &join.keys(), exec, join.suffix(), - &join.pending_order(), resolved_join_parallelism(exec)); - if (auto err = op->run_build()) { - return std::unexpected(std::move(*err)); - } - return op->take_fusible_probe(); -} - -auto probe_morsel_workers(const Table& input, const ExecutionContext& exec) -> std::size_t { - // OPT-IN, and the measurement is why. On its own this is a LOSS: the - // probe already fans out inside one chunk (`probe_ranges_parallel`) with - // no copy, and morselizing replaces that with a per-morsel gather plus an - // ordered merge. Measured at 8 cores, interleaved, median of 15: - // q05 +1.7%, q09 +6.8% -- the only two PDS-H queries where it fires. - // - // It is wired anyway because the gather is not what this shape is for. - // A probe morsel becomes worth its gather when the filters and - // projections above the join run in the SAME chain, so the join's output - // is never materialized between them. That needs the probe admitted into - // the plan's step vocabulary, which is the next piece; this is the half - // that had to work first, and it does -- 22/22 answers at 1, 2 and 8 - // cores with `IBEX_PROBE_MORSELS=1`. - if (std::getenv("IBEX_PROBE_MORSELS") == nullptr) { - return 0; - } - if (!exec.can_fan_out() || on_worker_pool_thread()) { - return 0; - } - if (!is_worth_morselizing(exec, input.rows(), input.columns.size())) { - return 0; - } - const std::size_t grain = morsel_grain(exec, input.rows()); - return morsel_worker_count(exec, partitioned_morsel_count(input, grain)); -} - -auto build_probe_morsel_pipeline(Table input, const JoinProbe& probe, std::size_t workers, - const ExecutionContext& exec) - -> std::expected { - auto owned = std::make_unique
(std::move(input)); - const std::size_t grain = morsel_grain(exec, owned->rows()); - const auto expected_morsels = partitioned_morsel_count(*owned, grain); - if (exec.parallel_stats != nullptr) { - exec.parallel_stats->parallel_pipelines.fetch_add(1, std::memory_order_relaxed); - exec.parallel_stats->morsels.fetch_add(expected_morsels, std::memory_order_relaxed); - } - - static const std::vector no_steps; - std::vector chains; - chains.reserve(workers); - for (std::size_t i = 0; i < workers; ++i) { - // The chain is source + probe. Each worker gets its OWN `JoinProbe` -- - // its scratch vectors and its per-chunk categorical head table are - // per-worker state -- and they share one build, because the index and - // the build side are both `shared_ptr` and have been since - // `f6a1a632` and `6df9a966`. Copying the probe is what that ownership - // was for. - auto worker = build_morsel_worker_chain(no_steps, *owned, nullptr, nullptr, exec, &probe); - if (!worker.has_value()) { - return std::unexpected(std::move(worker.error())); - } - chains.push_back(std::move(worker.value())); - } - return std::make_unique(std::move(owned), std::move(chains), grain, - expected_morsels, process_worker_pool()); -} - -} // namespace - -namespace { - -/// The steps of a plan's parallel prefix, ordered source-to-sink. A plan -/// records steps sink-first; every executor here composes bottom-up. -auto parallel_pipeline_operators(const physical::Plan& plan) -> std::vector { - std::vector operators; - operators.reserve(plan.parallel_step_count()); - for (std::size_t i = plan.parallel_end; i > plan.parallel_begin; --i) { - operators.push_back(plan.steps[i - 1]); - } - return operators; -} - -/// Run a physical map pipeline over morsels. The plan says which steps may run -/// in parallel (`parallel_steps`) and what feeds them (`parallel_input_node`); -/// this builds that input, materializes it, and executes the prefix over its -/// morsels. It is the pipeline's parallel *mode*, not a separate executor with -/// its own idea of what is eligible. -/// -/// `steps` inside a plan are sink-first; the operators here run source-to-sink, -/// so the prefix is reversed once, on the build thread. -auto build_map_pipeline_parallel(const physical::Plan& plan, const TableRegistry& registry, - const ScalarRegistry* scalars, const ExternRegistry* externs, - const ExecutionContext& exec, ModelResult* model_out) + const ExternRegistry* externs, const ExecutionContext& exec, + ModelResult* model_out, Fn fn) -> std::expected { - const std::vector operators = parallel_pipeline_operators(plan); - const ir::Node* input_node = physical::parallel_input_node(plan); - if (input_node == nullptr) { - return std::unexpected("map pipeline: parallel mode without an input"); - } - - // Source strategy, decided here because it is a property of this run's - // input rather than of the query's root. A decomposable deferred scan can - // feed the run one unit at a time -- decode and maps in the same worker - // task, the ordered ring feeding whatever is above -- instead of being - // decoded whole and morselized. Probe scans keep their join-owned dynamic - // filter timing (a null filter slot is what distinguishes them) and so do - // not stream here. - if (exec.stream_scans && input_node->kind() == ir::NodeKind::Scan) { - const auto& scan = ir::node_cast(*input_node); - if (!registry.contains(scan.source_name())) { - if (const auto* deferred = exec.deferred_scan(scan.source_name()); - deferred != nullptr && deferred->filter == nullptr) { - auto units = deferred_scan_units(*deferred); - if (units.size() > 1 && scan_pipeline_worker_count(units.size()) >= 2) { - return build_pipelined_scan(operators, true, *deferred, std::move(units), - scalars, externs, exec); - } - } - } - } - - // Fused probe: when this pipeline's input is a streaming join, take its - // build and its probe side and run the probe at the head of every worker - // chain, instead of materializing the join's OUTPUT and morselizing that. - // The probe side becomes the morsel source, so the join's output is never - // assembled as a table at all -- it is produced a morsel at a time by the - // same worker that then runs the maps over it. Two pipelines meeting at a - // barrier, which is the shape this plan has been working toward. - std::optional fused_probe; - std::unique_ptr
owned; - if (input_node->kind() == ir::NodeKind::Join) { - auto fused = try_take_join_probe(*input_node, registry, scalars, externs, exec, model_out); - if (!fused.has_value()) { - return std::unexpected(std::move(fused.error())); - } - if (fused->has_value()) { - owned = std::make_unique
(std::move((*fused)->probe_side)); - fused_probe = std::move((*fused)->probe); - } - } - if (owned == nullptr) { - auto input_op = build_operator(*input_node, registry, scalars, externs, exec, model_out); - if (!input_op.has_value()) { - return std::unexpected(std::move(input_op.error())); - } - auto input_tbl = materialize_operator(std::move(input_op.value())); - if (!input_tbl.has_value()) { - return std::unexpected(std::move(input_tbl.error())); - } - owned = std::make_unique
(std::move(input_tbl.value())); - } - const std::size_t grain = morsel_grain(exec, owned->rows()); - const auto expected_morsels = partitioned_morsel_count(*owned, grain); - // Morselize only when the work would actually fan out. Splitting earns its - // cost by having several workers share it; with fewer than two the split, - // the per-morsel gather and the merge concat are all paid for parallelism - // that was already ruled out, so the whole-table chunk below is the right - // shape. - // - // No time claim is attached to this: measured on its own it moved nothing - // (`morsels` 92 -> 0 on q12/q14, wall time unchanged), because by the time - // this runs the input has already been materialized. The run's real cost at - // a budget of one is that materialize, and it is declined at the - // construction seam in `build_physical_map_step` instead. What this does - // earn is an honest counter -- see below -- and not doing work whose only - // consumer is a worker that will not exist. - const std::size_t worker_count = - is_worth_morselizing(exec, owned->rows(), owned->columns.size()) - ? morsel_worker_count(exec, expected_morsels) - : 0; - const bool morselize = worker_count >= 2; - if (exec.parallel_stats != nullptr) { - auto& stats = *exec.parallel_stats; - (morselize ? stats.parallel_pipelines : stats.serial_pipelines) - .fetch_add(1, std::memory_order_relaxed); - if (morselize) { - // Only count morsels that are actually formed. Reporting the - // would-be split for a run that takes the whole-table chunk is what - // made this cost invisible: `morsels=115` on a query with - // `parallel=0` reads as work done, not as work paid for and thrown - // away. - stats.morsels.fetch_add(expected_morsels, std::memory_order_relaxed); - } - } - - if (worker_count >= 2) { - const auto head = (operators.empty() || fused_probe.has_value()) - ? std::nullopt - : range_filter_head(operators.front(), *owned); - if (exec.parallel_stats != nullptr && head.has_value()) { - exec.parallel_stats->range_heads.fetch_add(1, std::memory_order_relaxed); - } - - // A range-native filter can skip the merger entirely by presizing its - // output — see TwoPhaseFilterOperator. Anything above it must be - // metadata-only: a row-touching operator would need the per-morsel - // chunks the two-phase form does not produce, but Project and Rename - // copy no rows and so are simply run once over the finished output. - const auto tail = std::span{operators}.subspan(head.has_value() ? 1 : 0); - if (head.has_value() && std::ranges::all_of(tail, [](const MapStep& step) { - return is_metadata_only_node(step.node->kind()); - })) { - auto layout = build_filter_output_layout(*owned, head->project); - // A missing projected column is left to the ordered merger below, - // which reports it through the normal evaluation path. - if (layout.has_value() && filter_gather_is_thread_safe(*owned, layout->src_of_dst)) { - if (exec.parallel_stats != nullptr) { - exec.parallel_stats->two_phase_filters.fetch_add(1, std::memory_order_relaxed); - } - // The tail is metadata-only by the check above, so no step in - // it carries a fused partner; its nodes are the whole story. - std::vector tail_nodes; - tail_nodes.reserve(tail.size()); - for (const MapStep& step : tail) { - tail_nodes.push_back(step.node); - } - return std::make_unique( - std::move(owned), *head->predicate, head->project != nullptr, - std::move(tail_nodes), scalars, std::move(layout.value()), grain, - expected_morsels, worker_count, process_worker_pool()); - } - } - - std::vector workers; - workers.reserve(worker_count); - for (std::size_t i = 0; i < worker_count; ++i) { - auto worker = - build_morsel_worker_chain(operators, *owned, scalars, externs, exec, - fused_probe.has_value() ? &*fused_probe : nullptr); - if (!worker.has_value()) { - return std::unexpected(std::move(worker.error())); - } - workers.push_back(std::move(worker.value())); - } - return std::make_unique(std::move(owned), std::move(workers), grain, - expected_morsels, process_worker_pool()); - } - - if (!morselize) { - // Too little work to be worth splitting: run the chain over one - // whole-table chunk. This is the plain serial path — same map - // operators, same `preserve_empty_morsels = false`, one chunk in and - // one chunk out — so it costs exactly what not forming a pipeline costs. - // Morselizing here instead would add a per-morsel gather and a merge - // concat to buy parallelism that was already judged not worth having. - OperatorPtr serial = make_table_source(std::move(*owned)); - if (fused_probe.has_value()) { - // Not worth morselizing, but the probe was already taken from the - // join and there is nothing to give it back to: run it here, over - // the whole probe side, with the maps above it as before. - serial = std::make_unique(std::move(serial), *fused_probe); - } - for (const MapStep& op_node : operators) { - auto next = build_row_local_map_operator(op_node, std::move(serial), scalars, externs, - exec, false); - if (!next.has_value()) { - return std::unexpected("morsel pipeline: " + next.error()); - } - serial = std::move(next.value()); - } - return serial; - } - - OperatorPtr chain = std::make_unique(*owned, grain); - for (const MapStep& op_node : operators) { - auto next = - build_row_local_map_operator(op_node, std::move(chain), scalars, externs, exec, true); - if (!next.has_value()) { - // The plan's step vocabulary only admits row-local map kinds. - return std::unexpected("morsel pipeline: " + next.error()); - } - chain = std::move(next.value()); - } - - chain = std::make_unique(std::move(chain), expected_morsels, grain); - return std::make_unique(std::move(owned), std::move(chain)); -} - -/// Streams a deferred lazy scan one source unit at a time instead of decoding -/// the whole source and handing it over as a single chunk. -/// -/// This is Phase 1 of `plans/pipelined-execution-plan.md`. The decode it -/// performs is the same decode `materialize_deferred_scan` performs, with the -/// same pushdowns — projection, static conjuncts, the dynamic key membership -/// filter, and both fused scans, all restricted to the unit rather than -/// declined (see `LazyTable::project_where_unit`). What changes is only that -/// the rows arrive in pieces, which is the precondition for anything above the -/// scan ever running concurrently with it. -/// -/// The plan is fixed once, at construction, for the reason `DeferredScanPlan` -/// documents: re-reading the shared filter slot per unit could apply to unit 3 -/// a bound that units 0-2 never saw. -/// -/// **Phase 2 (concurrent units).** Units are decoded a WINDOW at a time on -/// worker threads rather than one after another, and the window after the one -/// being served is already decoding. Phase 1 measured why: decoding units -/// serially cut total work (pool work on q01 fell 234ms -> 125ms) but raised -/// 8-core wall, because the pool saw one short burst per unit with a serial -/// phase between and occupancy fell to 0.14. Nothing was too small to -/// parallelize — there was just never more than one unit's worth of work -/// available at a time. -/// -/// Decoding a unit on a worker is safe, and specifically so: -/// -/// * `LazyTable::acquire_reader` hands each concurrent acquisition its OWN -/// reader product, under a mutex, and a product owns all the mutable -/// decoder state. That is exactly what the reader pool was built for. -/// * `project_where_unit` never WRITES `cache_` — a unit holds a fragment of -/// a column, so it must not — and concurrent reads of it are fine. This is -/// load-bearing now, not just a correctness nicety: routing any part of the -/// unit path back through `project()`, which does cache, would turn this -/// into a data race. -/// * Every inner parallel path (`parallel_readers`, `for_row_ranges`, -/// `filter_selection`) checks `on_worker_pool_thread()` and runs serial -/// inside a task, so the outer window is the only level of parallelism and -/// nothing submits from a worker into a saturated pool. -/// -/// Ordering is preserved exactly: workers claim units from a shared cursor and -/// write only their own slot, and chunks are served in unit order with -/// `sequence` / `row_offset` assigned on the calling thread. The categorical -/// remap also stays on the calling thread, in unit order, because it folds each -/// chunk into a dictionary shared with every earlier one. -class DeferredScanSourceOperator final : public Operator { - public: - DeferredScanSourceOperator(const DeferredScan& scan, std::vector units, - const ExecutionContext& exec) - : scan_(&scan), - plan_(plan_deferred_scan(scan)), - units_(std::move(units)), - exec_(&exec), - window_(unit_window(exec)) {} - - [[nodiscard]] auto next() -> std::expected, std::string> override { - while (true) { - if (served_ < ready_.size()) { - auto& slot = ready_[served_++]; - if (!slot.has_value()) { - return std::unexpected(std::move(slot.error())); - } - auto chunk = emit(std::move(*slot)); - if (!chunk.has_value()) { - continue; // the unit's rows were all filtered out - } - return std::optional{std::move(*chunk)}; - } - // Out of decoded units: put the next window in flight and wait for - // it. - // - // Dispatching the window AFTER this one here too, so the consumer's - // work would overlap the next decode, is a MEASURED DEAD END. It - // doubles how much of the source is decoded at once — peak RSS on a - // 25-row-group scan went 161MB -> 244MB — and returns nothing, - // because the consumer is a blocking operator that eats chunks - // faster than they decode, so there is no consumer work to overlap - // with. Real overlap needs a pipeline that keeps running while the - // scan decodes, which is the rest of Phase 2, not a deeper queue. - // - // `inflight_pending_`, not `batch_`: a one-unit window decodes on - // this thread and never submits, so a `batch_`-based test reports - // "nothing in flight" and drops a window that has already been - // decoded. That silently lost every unit after the first whenever - // the query was not parallel, and every trailing single-unit window - // when it was. - if (!inflight_pending_) { - if (dispatched_ >= units_.size()) { - // A stream must still carry its schema when every unit was - // empty after scan pushdown. Without this carrier, - // MaterializeOperator sees end-of-stream as its first - // result and returns a column-less Table, unlike the - // equivalent eager scan (and unlike a filter above an - // ordinary table source). Keep the first empty unit out - // of the normal pipeline -- several streaming operators - // intentionally do not consume empty chunks -- and emit it - // only when it is the sole result. - if (empty_schema_carrier_.has_value()) { - auto carrier = std::move(*empty_schema_carrier_); - empty_schema_carrier_.reset(); - return std::optional{emit_schema_carrier(std::move(carrier))}; - } - return std::optional{}; - } - dispatch(); - } - harvest(); - } - } - - private: - /// How many units to decode at once. One means serial, which is what a - /// non-parallel query, a single-thread budget, and a call already running - /// inside a pool task all get — the last because submitting from a worker - /// deadlocks against a saturated pool. - static auto unit_window(const ExecutionContext& exec) -> std::size_t { - if (!exec.can_fan_out() || on_worker_pool_thread()) { - return 1; - } - auto& pool = process_worker_pool(); - const std::size_t budget = exec.compute_budget(); - return std::max(1, std::min(budget, pool.size())); - } - - /// Put the next window of units in flight. Returns without blocking: - /// `WorkerPool::submit` is asynchronous, which is what makes the overlap - /// possible — and which is also why everything the body touches is a - /// member rather than a local. - void dispatch() { - window_begin_ = dispatched_; - window_count_ = std::min(window_, units_.size() - dispatched_); - dispatched_ += window_count_; - inflight_.clear(); - inflight_.resize(window_count_); - inflight_pending_ = true; - if (window_count_ == 1) { - // Nothing to overlap and no reason to pay for a pool round trip. - inflight_[0] = decode_unit(0); - return; - } - cursor_.store(0, std::memory_order_relaxed); - batch_ = process_worker_pool().submit(window_count_, [this](std::size_t /*worker*/) { - for (std::size_t i = cursor_.fetch_add(1, std::memory_order_relaxed); i < window_count_; - i = cursor_.fetch_add(1, std::memory_order_relaxed)) { - inflight_[i] = decode_unit(i); - } - }); - } - - /// One unit's decode, as run by a worker. Never throws out of the body: a - /// `Batch` rethrows the first escaped exception at `wait()`, which would - /// lose the other units' errors and unwind through the pool. - auto decode_unit(std::size_t slot) -> std::expected { - try { - auto table = - materialize_deferred_scan_unit(*scan_, plan_, units_[window_begin_ + slot], *exec_); - if (table.has_value()) { - normalize_time_index(*table); - } - return table; - } catch (const std::exception& e) { - return std::unexpected(std::string("streamed scan: ") + e.what()); - } - } - - void harvest() { - if (batch_.has_value()) { - batch_->wait(); - batch_.reset(); - } - ready_ = std::move(inflight_); - inflight_.clear(); - inflight_pending_ = false; - served_ = 0; - } - - /// Turn a decoded unit into the chunk to hand upward, or nullopt when it - /// carries no rows. - /// - /// A unit whose every row the scan's predicates rejected carries nothing. - /// Skipping it is not just an optimization: an empty chunk with columns is - /// a shape some operators would rather not meet, and dropping it changes no - /// result. A column-less chunk is a different thing — it carries a row - /// count for `count()` — and is kept. - auto emit(Table table) -> std::optional { - if (!table.columns.empty() && table.rows() == 0) { - if (!empty_schema_carrier_.has_value()) { - empty_schema_carrier_ = std::move(table); - } - return std::nullopt; - } - // A non-empty result makes a deferred carrier unnecessary. - empty_schema_carrier_.reset(); - unify_categorical_dictionaries(table); - Chunk chunk; - const std::size_t rows = table.rows(); - chunk.set_properties(table.properties()); - chunk.columns = std::move(table.columns); - if (chunk.columns.empty()) { - chunk.logical_rows = table.logical_rows; - } - chunk.sequence = sequence_++; - chunk.row_offset = emitted_rows_; - emitted_rows_ += rows; - return chunk; - } - - /// Materialize the one schema carrier retained when scan pushdown rejected - /// every row. It follows the ordinary source identity convention: first - /// chunk, at row zero. Dictionary unification remains necessary because a - /// zero-row categorical still carries dictionary identity as part of its - /// schema. - auto emit_schema_carrier(Table table) -> Chunk { - unify_categorical_dictionaries(table); - Chunk chunk; - chunk.set_properties(table.properties()); - chunk.columns = std::move(table.columns); - if (chunk.columns.empty()) { - chunk.logical_rows = table.logical_rows; - } - chunk.sequence = sequence_++; - chunk.row_offset = emitted_rows_; - return chunk; - } - - /// Remap every Categorical column onto a dictionary shared by all this - /// source's chunks. - /// - /// Parquet writes one dictionary PER ROW GROUP, and a unit is one row - /// group, so without this each chunk's codes would mean something different - /// from the last one's — and the operators that compare dictionary identity - /// to take a fast path (grouping, joins, the packed key encoder) would be - /// comparing codes across dictionaries that disagree. The whole-file decode - /// never had this problem because it merged the groups' dictionaries - /// itself. `ChunkedParquetSourceOperator` solves it the same way. - /// - /// **One lookup per dictionary ENTRY, never per row.** Interning row by row - /// is a string hash per row, and it does not announce itself: TPC-H's - /// `l_returnflag` and `l_linestatus` are plain `string` in the Arrow schema - /// and only become Categorical because the writer dictionary-encoded them, - /// so a query that never mentions a categorical type still pays. Measured - /// on q01, per-row interning cost 114ms of the scan's 160ms — the entire - /// regression against the materialized path, on the calling thread where - /// nothing could overlap it. A dictionary has a handful of entries and a - /// unit has a million rows; the difference is the whole cost. - void unify_categorical_dictionaries(Table& table) { - using code_type = Column::code_type; - for (std::size_t i = 0; i < table.columns.size(); ++i) { - auto* local = std::get_if>(table.columns[i].column.get()); - if (local == nullptr) { - continue; - } - if (cat_states_.size() <= i) { - cat_states_.resize(table.columns.size()); - } - auto& state = cat_states_[i]; - if (!state.has_value()) { - state.emplace(); - } - if (state->dictionary_ptr() == local->dictionary_ptr()) { - continue; // already speaks the shared dictionary - } - // Intern this chunk's dictionary into the shared one, reading back - // the code each entry landed on. `clear()` drops the codes and - // keeps the dictionary, which is exactly what an accumulator wants. - const auto& dictionary = local->dictionary(); - state->clear(); - for (const auto& value : dictionary) { - state->push_back(value); - } - std::vector remap(dictionary.size()); - for (std::size_t entry = 0; entry < dictionary.size(); ++entry) { - remap[entry] = state->code_at(entry); - } - state->clear(); - - const auto& local_codes = local->codes(); - std::vector codes(local_codes.size()); - for (std::size_t row = 0; row < local_codes.size(); ++row) { - codes[row] = remap[static_cast(local_codes[row])]; - } - table.columns[i].column = std::make_shared( - Column{state->dictionary_ptr(), state->index_ptr(), std::move(codes)}); - } + auto materialized = + materialize_row_local(child_node, registry, scalars, externs, exec, model_out); + if (!materialized.has_value()) { + return std::unexpected(std::move(materialized.error())); } - - const DeferredScan* scan_; - DeferredScanPlan plan_; - std::vector units_; - const ExecutionContext* exec_; - std::vector>> cat_states_; - - /// Units decoded and waiting to be served, in unit order, and how many of - /// them have been. - std::vector> ready_; - std::size_t served_ = 0; - /// First empty, column-bearing unit. It becomes a schema carrier only if - /// every unit was empty; otherwise empty units stay invisible to the - /// streaming operators above this source. - std::optional
empty_schema_carrier_; - /// The window currently being decoded. Workers write disjoint slots of - /// this, so it must not be resized while `batch_` is live. - std::vector> inflight_; - std::optional batch_; - /// Whether `inflight_` holds a dispatched window awaiting harvest. Not - /// derivable from `batch_`: a one-unit window is decoded inline. - bool inflight_pending_ = false; - std::atomic cursor_{0}; - std::size_t window_begin_ = 0; - std::size_t window_count_ = 0; - - std::size_t dispatched_ = 0; - std::size_t emitted_rows_ = 0; - std::uint64_t sequence_ = 0; - std::size_t window_ = 1; -}; - -// A one-chunk source owned by one scan-pipeline worker. The worker replaces -// the pending chunk for every source unit it claims, then pulls the private -// row-local chain exactly once. Keeping the chain private is what makes its -// mutable per-operator state safe without locks. -class ScanPipelineSource final : public Operator { - public: - void set(Chunk chunk) { pending_ = std::move(chunk); } - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (!pending_.has_value()) { - return std::optional{}; - } - auto chunk = std::move(*pending_); - pending_.reset(); - return std::optional{std::move(chunk)}; + auto result = fn(std::move(materialized.value())); + if (!result.has_value()) { + return std::unexpected(std::move(result.error())); } - - private: - std::optional pending_; -}; - -struct ScanPipelineWorker { - ScanPipelineSource* source = nullptr; - OperatorPtr chain; -}; - -[[nodiscard]] auto build_scan_pipeline_worker(const std::vector& operators, - const ScalarRegistry* scalars, - const ExternRegistry* externs, - const ExecutionContext& exec) - -> std::expected { - auto source = std::make_unique(); - ScanPipelineWorker worker{.source = source.get(), .chain = std::move(source)}; - for (const MapStep& op_node : operators) { - auto next = build_row_local_map_operator(op_node, std::move(worker.chain), scalars, externs, - exec, true); - if (!next.has_value()) { - return std::unexpected("scan pipeline: " + next.error()); - } - worker.chain = std::move(next.value()); - } - return worker; + return make_table_source(std::move(result.value())); } -/// A bounded source-to-map pipeline. -/// -/// Each worker claims one source unit, decodes it with all globally planned -/// pushdowns intact, immediately runs the row-local operator chain, and -/// publishes the result into a bounded ordered ring. The caller drains that -/// ring into the next blocking operator. Thus decode of unit N+1, row-local -/// work on unit N, and consumption of an earlier unit can all be live at once; -/// there is no materialized table or whole-window wait between those stages. -class PipelinedScanOperator final : public Operator { - public: - PipelinedScanOperator(const DeferredScan& scan, std::vector units, - std::vector workers, const ExecutionContext& exec, - WorkerPool& pool) - : scan_(&scan), - plan_(plan_deferred_scan(scan)), - units_(std::move(units)), - workers_(std::move(workers)), - exec_(&exec), - pool_(&pool), - window_(std::max(workers_.size() * 2, 2)), - ring_(window_, workers_.size()) {} - - ~PipelinedScanOperator() override { cancel_and_join(); } - - PipelinedScanOperator(const PipelinedScanOperator&) = delete; - auto operator=(const PipelinedScanOperator&) -> PipelinedScanOperator& = delete; - PipelinedScanOperator(PipelinedScanOperator&&) = delete; - auto operator=(PipelinedScanOperator&&) -> PipelinedScanOperator& = delete; - - [[nodiscard]] auto next() -> std::expected, std::string> override { - if (finished_) { - return std::optional{}; - } - start(); - - while (next_sequence_ < units_.size()) { - std::optional produced = ring_.take(next_sequence_); - if (!produced.has_value()) { - // Stopped before this unit: cancelled, failed, or out of - // workers. The ring reports the lowest-sequence failure, so the - // message does not depend on which worker lost a race. - if (auto failure = ring_.failure(); failure.has_value()) { - return fail(std::move(*failure)); - } - return fail(interrupt_requested() ? interrupt_message() - : "scan pipeline: missing output unit"); - } - ++next_sequence_; - Chunk chunk = std::move(*produced); - if (!chunk.columns.empty() && chunk.rows() == 0) { - if (!empty_schema_carrier_.has_value()) { - empty_schema_carrier_ = std::move(chunk); - } - continue; - } - - empty_schema_carrier_.reset(); - normalize_categorical_dictionaries(chunk); - restamp(chunk); - return std::optional{std::move(chunk)}; - } - - finish_workers(); - finished_ = true; - if (empty_schema_carrier_.has_value()) { - auto carrier = std::move(*empty_schema_carrier_); - empty_schema_carrier_.reset(); - normalize_categorical_dictionaries(carrier); - restamp(carrier); - return std::optional{std::move(carrier)}; - } - return std::optional{}; - } - - private: - void start() { - if (started_) { - return; - } - started_ = true; - batch_ = pool_->submit(workers_.size(), [this](std::size_t id) { run_worker(id); }); - } - - void run_worker(std::size_t worker_id) noexcept { - // However this worker leaves, it must stop counting as a producer, or - // the consumer waits for a unit that is never coming. - // One-shot scope guard: aggregate-initialised, never copied or moved. - // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions) - struct ExitGuard { - OrderedChunkRing* ring; - ~ExitGuard() { ring->producer_exited(); } - } const guard{&ring_}; - - std::size_t claimed = 0; - try { - auto& worker = workers_[worker_id]; - while (true) { - const std::size_t sequence = cursor_.fetch_add(1, std::memory_order_relaxed); - if (sequence >= units_.size()) { - return; - } - claimed = sequence; - if (ring_.acquire(sequence) == OrderedChunkRing::Acquire::Abandon) { - return; - } - if (interrupt_requested()) { - ring_.cancel(); - return; - } +} // namespace - auto result = run_unit(worker, sequence); - if (!result.has_value()) { - ring_.record_error(sequence, std::move(result.error())); - return; - } - ring_.publish(sequence, std::move(*result)); - } - } catch (const std::exception& error) { - // Sequence-tagged like any other failure, so the reported message - // obeys the same lowest-sequence rule. Composing it allocates and - // the likeliest exception here is bad_alloc, hence the - // allocation-free fallback underneath. - try { - ring_.record_error(claimed, - "scan pipeline: worker exception: " + std::string(error.what())); - } catch (...) { - ring_.record_fault(claimed, - "scan pipeline: worker exception (no memory to report it)"); - } - } catch (...) { - ring_.record_fault(claimed, "scan pipeline: worker threw a non-standard exception"); +auto make_join_probe_operator(OperatorPtr source, std::optional
materialized_source, + JoinProbeFactory probe) -> std::expected { + const ExecutionContext* exec = probe.execution_context(); + if (materialized_source.has_value() && exec != nullptr) { + if (const std::size_t workers = + pipeline_executor_detail::probe_morsel_workers(*materialized_source, *exec); + workers >= 2) { + return pipeline_executor_detail::build_probe_morsel_pipeline( + std::move(*materialized_source), probe, workers, *exec); } } - - [[nodiscard]] auto run_unit(ScanPipelineWorker& worker, std::size_t sequence) - -> std::expected { - auto decoded = materialize_deferred_scan_unit(*scan_, plan_, units_[sequence], *exec_); - if (!decoded.has_value()) { - return std::unexpected(std::move(decoded.error())); - } - normalize_time_index(*decoded); - worker.source->set(table_to_chunk( - std::move(*decoded), - ChunkIdentity{.sequence = sequence, .row_offset = units_[sequence].start})); - auto produced = worker.chain->next(); - if (!produced.has_value()) { - return std::unexpected(std::move(produced.error())); - } - if (!produced->has_value()) { - return std::unexpected("scan pipeline: row-local chain dropped a source unit"); - } - Chunk chunk = std::move(**produced); - if (chunk.sequence != sequence) { - return std::unexpected("scan pipeline: row-local chain reordered a source unit"); - } - return chunk; + OperatorPtr probe_source = materialized_source.has_value() + ? make_table_source(std::move(*materialized_source)) + : std::move(source); + if (probe_source == nullptr) { + return std::unexpected("join probe has no probe-side source"); } + return probe.attach_move(std::move(probe_source)); +} - void restamp(Chunk& chunk) noexcept { - chunk.sequence = emitted_sequence_++; - chunk.row_offset = emitted_rows_; - emitted_rows_ += chunk.rows(); - } +namespace { - // Parquet dictionaries are local to row groups. Ordered publication is the - // one serial point where chunks are remapped onto one shared dictionary, - // preserving the existing streamed-source contract for downstream keys. - void normalize_categorical_dictionaries(Chunk& chunk) { - using code_type = Column::code_type; - if (cat_states_.size() < chunk.columns.size()) { - cat_states_.resize(chunk.columns.size()); - } - for (std::size_t i = 0; i < chunk.columns.size(); ++i) { - auto* local = std::get_if>(chunk.columns[i].column.get()); - if (local == nullptr) { - continue; - } - auto& state = cat_states_[i]; - if (!state.has_value()) { - state.emplace(); - } - if (state->dictionary_ptr() == local->dictionary_ptr()) { - continue; - } - const auto& dictionary = local->dictionary(); - state->clear(); - for (const auto& value : dictionary) { - state->push_back(value); - } - std::vector remap(dictionary.size()); - for (std::size_t entry = 0; entry < dictionary.size(); ++entry) { - remap[entry] = state->code_at(entry); - } - state->clear(); - const auto& local_codes = local->codes(); - std::vector codes(local_codes.size()); - for (std::size_t row = 0; row < local_codes.size(); ++row) { - codes[row] = remap[static_cast(local_codes[row])]; - } - chunk.columns[i].column = std::make_shared( - Column{state->dictionary_ptr(), state->index_ptr(), std::move(codes)}); - } - } +// A materializing binary breaker (non-streamable join, matmul) now resolves in +// `interpret_node`, which drains both sides whole-table and serially. +// Overlapping the two materializations on a raw std::thread was tried twice +// (once unbudgeted, once under a since-removed helper-thread budget) and +// regressed the PDS-H suite both times (q09 +57% / +47.5%): the cost is +// structural -- both sides are already-expensive full materializations +// contending for the same cores/bandwidth -- not a concurrency count a budget +// could bound. A future attempt needs a cost-aware gate (skip when both sides +// are large), and belongs wherever that breaker is lifted onto the physical plan. - void finish_workers() { - if (batch_.has_value()) { - batch_->wait(); - batch_.reset(); - } - if (validated_) { - return; +auto eval_extern_args(const std::vector& exprs, const ScalarRegistry* scalars, + const ExternRegistry* externs) -> std::expected { + ExternArgs args; + args.reserve(exprs.size()); + for (const auto& arg : exprs) { + auto val = eval_expr(arg, Table{}, 0, scalars, externs); + if (!val.has_value()) { + return std::unexpected(std::move(val.error())); } - validated_ = true; - for (auto& worker : workers_) { - auto trailing = worker.chain->next(); - if (!trailing.has_value()) { - throw std::runtime_error(std::move(trailing.error())); - } - if (trailing->has_value()) { - throw std::runtime_error("scan pipeline: unexpected trailing output"); - } + // Externs take null-free ScalarValue arguments (see the null-arm plan). + auto scalar = scalar_from_expr(val.value()); + if (!scalar.has_value()) { + return std::unexpected("null argument in extern function call"); } + args.push_back(std::move(*scalar)); } + return args; +} + +} // namespace - void cancel_and_join() noexcept { - ring_.cancel(); - try { - finish_workers(); - } catch (...) { // NOLINT(bugprone-empty-catch) +auto invoke_extern_call(const ir::ExternCallNode& ec, const ScalarRegistry* scalars, + const ExternRegistry* externs) -> std::expected { + if (externs == nullptr) { + return std::unexpected("extern call with no registry: " + ec.callee()); + } + const auto* fn = externs->find(ec.callee()); + if (fn == nullptr) { + return std::unexpected("unknown extern function: " + ec.callee()); + } + if (fn->first_arg_is_table) { + return std::unexpected("extern function requires a table input: " + ec.callee()); + } + auto args = eval_extern_args(ec.args(), scalars, externs); + if (!args.has_value()) { + return std::unexpected(std::move(args.error())); + } + if (fn->kind == ExternReturnKind::Table && fn->chunked_table_func) { + auto source = fn->chunked_table_func(args.value()); + if (source.has_value()) { + auto materialized = materialize_operator(std::move(source.value())); + if (!materialized.has_value()) { + return std::unexpected(std::move(materialized.error())); + } + return ExternValue{std::move(materialized.value())}; } } - - [[nodiscard]] auto fail(std::string message) - -> std::expected, std::string> { - finished_ = true; - cancel_and_join(); - return std::unexpected(std::move(message)); + auto result = fn->func(args.value()); + if (!result.has_value()) { + return std::unexpected(std::move(result.error())); } + return result; +} - const DeferredScan* scan_; - DeferredScanPlan plan_; - std::vector units_; - std::vector workers_; - const ExecutionContext* exec_; - WorkerPool* pool_; - std::size_t window_ = 2; - // The same ordered handoff the morsel executor uses: one implementation of - // the bounded, sequence-ordered producer/consumer shape. - OrderedChunkRing ring_; - std::vector>> cat_states_; - std::optional empty_schema_carrier_; - std::optional batch_; - std::atomic cursor_{0}; - std::size_t next_sequence_ = 0; - std::size_t emitted_rows_ = 0; - std::uint64_t emitted_sequence_ = 0; - bool started_ = false; - bool finished_ = false; - bool validated_ = false; -}; - -/// A bounded asynchronous boundary between two pipeline segments. -/// -/// The existing executor is pull-based, which is ideal for operator-local -/// state but normally means a parent cannot start its work until its child has -/// returned from `next()`. This stage retains that contract at both ends while -/// driving its child on a dedicated scheduler thread and holding at most two -/// ordered chunks between them. A breaker below the stage may therefore build -/// or probe the next chunk while a row-local parent (or the next breaker) is -/// working on the preceding one. -/// -/// This deliberately does not borrow a WorkerPool thread. A streamed scan -/// already owns pool tasks, and putting the stage on that same fixed pool -/// reintroduces the saturated-pool deadlock that the scan producer's worker -/// reservation avoids. The query lease limits this to one query, while the -/// builder only inserts stages at breaker boundaries, so this is bounded by -/// plan depth rather than morsel count. -class PipelinedStageOperator final : public Operator { - public: - PipelinedStageOperator(OperatorPtr child, ExecutionProfileEntry* entry) - : child_(std::move(child)), entry_(entry) {} - - ~PipelinedStageOperator() override { cancel_and_join(); } - - PipelinedStageOperator(const PipelinedStageOperator&) = delete; - auto operator=(const PipelinedStageOperator&) -> PipelinedStageOperator& = delete; - PipelinedStageOperator(PipelinedStageOperator&&) = delete; - auto operator=(PipelinedStageOperator&&) -> PipelinedStageOperator& = delete; - - [[nodiscard]] auto next() -> std::expected, std::string> override { - start(); - std::expected, std::string> result = std::optional{}; - { - std::unique_lock lock(mutex_); - { - // Idle, not serial work: waiting on the stage's producer thread. - const RingWaitScope ring_wait; - ready_.wait(lock, [this] { - return !ready_chunks_.empty() || producer_done_ || failure_.has_value(); - }); - } - if (failure_.has_value()) { - result = std::unexpected(std::move(*failure_)); - failure_.reset(); - } else if (!ready_chunks_.empty()) { - result = std::optional{std::move(ready_chunks_.front())}; - ready_chunks_.pop_front(); - } else { - done_ = true; - } +auto execute_program_preamble(const std::vector& preamble, + const ScalarRegistry* scalars, const ExternRegistry* externs) + -> std::expected { + for (const auto& node : preamble) { + if (node->kind() != ir::NodeKind::ExternCall) { + return std::unexpected("program preamble only supports extern calls"); } - space_.notify_one(); + const auto& ec = ir::node_cast(*node); + auto result = invoke_extern_call(ec, scalars, externs); if (!result.has_value()) { - cancel_and_join(); - return result; - } - if (done_) { - join(); + return std::unexpected(std::move(result.error())); } - return result; } + return {}; +} - private: - static constexpr std::size_t kCapacity = 2; +/// Planner seam: returns a pull-based operator that, when drained, +/// produces the logical result of `node`. Chunked operators exist +/// today for node kinds that are safe and useful to stream; any other +/// node kind falls back to the full-table `interpret_node` path and +/// is wrapped in a `TableSourceOperator` so downstream chunked +/// operators see a uniform pull-based interface. +// Order-delay past Filter/Project/Rename, and Head/Tail pushdown past +// Project/Rename, are handled by the IR canonicalize pass +// (src/ir/canonicalize.cpp). IR arrives here in canonical form, so +// build_operator only needs one branch per NodeKind and the shapes it +// matches are the post-canonicalization shapes (e.g. Project(Filter(x)) +// for the fused operator, not Project(Filter(Order(x)))). - void start() { - if (started_) { - return; - } - started_ = true; - producer_ = std::thread([this] { - // Declare what kind of thread this is. It is deliberately NOT a pool - // worker — it is long-lived and parks on the consumer's ring - // backpressure, which a fixed-size pool cannot host — but it must - // still be countable, and the profiler must not charge its work to - // the calling thread's self time. - const StageThreadScope stage_thread; - produce(); - }); +auto morsel_grain(const ExecutionContext& exec, std::size_t rows) -> std::size_t { + if (exec.parallel_grain != 0) { + return exec.parallel_grain; // explicit override, used as given } + // Aim for this many morsels per worker, so one slow morsel cannot strand + // the others. Below ~2 per thread the sweep shows real imbalance loss. + constexpr std::size_t kMorselsPerThread = 4; + // The measured good band was 16k-256k; clamp inside it. The upper bound is + // what stops the formula from choosing a grain worse than the old constant + // on a large input — see the declaration. + constexpr std::size_t kMinGrain = 4096; + constexpr std::size_t kMaxGrain = 65536; - void produce() noexcept { - try { - while (true) { - { - std::unique_lock lock(mutex_); - // Backpressure: the producer has filled the ring and the - // consumer has not drained it. Idle time on a runtime-owned - // thread, and the mirror image of the consumer's park at the - // other end of the same ring — a large value here means the - // CONSUMER is the bottleneck. `RingWaitScope` routes it to - // this thread's stage ledger. - const RingWaitScope ring_wait; - space_.wait(lock, - [this] { return cancelled_ || ready_chunks_.size() < kCapacity; }); - if (cancelled_) { - return; - } - } - - auto next = [&] { - // Scoped per pull, mirroring ProfiledOperator, so the - // backpressure wait above stays outside it. - const ExecutionProfileScope scope(entry_, ProfilePhase::Next); - return child_->next(); - }(); - if (!next.has_value()) { - fail(std::move(next.error())); - return; - } - if (!next->has_value()) { - { - const std::scoped_lock lock(mutex_); - producer_done_ = true; - } - ready_.notify_all(); - return; - } + const std::size_t pool_size = process_worker_pool().size(); + const std::size_t budget = exec.compute_budget(); + const std::size_t threads = std::max(std::min(budget, pool_size), 1); + return std::clamp(rows / (threads * kMorselsPerThread), kMinGrain, kMaxGrain); +} - { - const std::scoped_lock lock(mutex_); - if (cancelled_) { - return; - } - // There is one producer. The capacity check immediately - // before `child_->next()` therefore reserves this slot: - // only the consumer can change the queue size meanwhile. - ready_chunks_.push_back(std::move(**next)); - } - ready_.notify_one(); +auto process_pipeline_stats() -> ParallelPipelineStats* { + // File-local, like the worker pool and the query lease: a bundled plugin + // statically links runtime code, so an inline header variable would give + // each plugin its own counter (the RTLD_LOCAL trap). + // + // The reporter is a separate static whose destructor runs at exit. It holds + // no reference to the counters it prints beyond the function-local statics + // above it, which outlive it by declaration order. + static const bool enabled = std::getenv("IBEX_PARALLEL_STATS") != nullptr; + static ParallelPipelineStats stats; + // Function-local exit reporter, instantiated once below; never copied or moved. + // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions) + struct Reporter { + ~Reporter() { + if (!enabled) { + return; } - } catch (const std::exception& error) { - fail("pipeline stage: producer exception: " + std::string(error.what())); - } catch (...) { - fail("pipeline stage: producer threw a non-standard exception"); - } - } - - void fail(std::string message) noexcept { - { - const std::scoped_lock lock(mutex_); - if (!failure_.has_value()) { - failure_ = std::move(message); + // Exit-time diagnostic: a failed stderr write must not turn into a + // `std::terminate` from a throwing destructor. + try { + ibex::formatting::print( + stderr, + "pipeline stats: parallel={} serial={} morsels={} " + "pipelined_scans={} pipelined_stages={} range_heads={} two_phase={} " + "parallel_fields={} parallel_direct_numeric_fields={} parallel_probes={} " + "parallel_hash_builds={} parallel_aggregate_partitions={} " + "parallel_aggregate_finalizes={} " + "grouped_lifted_group_state={} chunk_direct_updates={}\n", + stats.parallel_pipelines.load(), stats.serial_pipelines.load(), + stats.morsels.load(), stats.pipelined_scans.load(), + stats.pipelined_stages.load(), stats.range_heads.load(), + stats.two_phase_filters.load(), stats.parallel_fields.load(), + stats.parallel_direct_numeric_fields.load(), stats.parallel_probes.load(), + stats.parallel_hash_builds.load(), stats.parallel_aggregate_partitions.load(), + stats.parallel_aggregate_finalizes.load(), + stats.grouped_lifted_group_state.load(), stats.chunk_direct_updates.load()); + } catch (...) { // NOLINT(bugprone-empty-catch) } - producer_done_ = true; } - ready_.notify_all(); + }; + static const Reporter reporter; + return enabled ? &stats : nullptr; +} + +void configure_parallel_from_env(ExecutionContext& exec) { + // The other two execution switches, applied the same way and for the same + // reason: one authority per setting. Both were `getenv` at their use sites + // until the seams that share them started to outnumber the settings — + // `stream_scans` is read at three build seams that must agree, and + // `parallel_join_probe` at three probe gates. An unset variable leaves the + // caller's choice alone, so a context built by hand still means "ignore the + // environment". + if (const auto want = stream_scans_from_env(); want.has_value()) { + exec.stream_scans = *want; + } + if (const auto want = parallel_join_probe_from_env(); want.has_value()) { + exec.parallel_join_probe = *want; + } + // Pin the COMPUTE budget to the core count. Every compute gate sizes itself + // from `parallel_threads`, falling back to the pool size when it is 0 — and + // the pool is now sized for decode, which wants more threads than cores + // (`decode_oversubscribe`). Without this, growing the pool would + // oversubscribe the compute paths too, and those measurably do not want it + // (q01 +4.6%, q17 +3.2%). With a multiplier of 1 this is the same number the + // fallback produced, so the default configuration is unchanged. + if (exec.parallel_threads == 0) { + exec.parallel_threads = compute_thread_count(); + } + if (exec.parallel_stats == nullptr) { + exec.parallel_stats = process_pipeline_stats(); + } + if (exec.execution_profile == nullptr && execution_profile_requested()) { + // The budget occupancy is measured against. Read from the context or + // the environment rather than `process_worker_pool().size()`, so that + // asking for a profile never constructs a pool a serial query would + // otherwise never have built. + const std::size_t budget = exec.compute_budget(); + exec.execution_profile = std::make_shared(budget); + } + if (const std::size_t grain = morsel_rows_from_env(); grain > 0) { + exec.parallel_grain = grain; + // An explicit grain is an explicit request to partition at that size, + // so it also lowers the serial threshold — otherwise the default + // threshold would silently override the knob it was asked to honor. + exec.parallel_min_rows = std::min(exec.parallel_min_rows, grain); } + // `parallel_threads` is set at the top of this function, not here: it is + // the COMPUTE budget and must track the core count, because the pool it + // used to default to is now sized for decode instead. +} + +namespace { - void join() noexcept { - if (producer_.joinable()) { - producer_.join(); +// One construction point for every row-local map operator that can live in a +// morsel pipeline. The serial planner uses the same factory: only the pipeline +// asks maps to retain zero-row morsels, because its ordered merger needs one +// output identity for every input morsel. Keeping the construction (especially +// FUP's gather set) here prevents the two planners from drifting as range-aware +// kernels replace these chunked implementations. +auto physical_filter_route(const ir::Expr& predicate, + const std::vector* source_signature) + -> kernel::FilterChunkRoute { + // A populated signature is the physical planner's proof that this is a + // registered table scan with one of the representations the Chunk gather + // family owns. Lazy/extern sources intentionally arrive without that + // proof and retain the compatibility route. + if (source_signature == nullptr || source_signature->empty() || + !kernel::supports_native_chunk_predicate(predicate)) { + return kernel::FilterChunkRoute::Auto; + } + for (const auto& signature : *source_signature) { + switch (signature.representation) { + case ColumnRepresentation::FixedWidth: + case ColumnRepresentation::PackedBool: + case ColumnRepresentation::StringSlabs: + case ColumnRepresentation::CategoricalCodes: + break; + default: + return kernel::FilterChunkRoute::Auto; } - } - - void cancel_and_join() noexcept { - { - const std::scoped_lock lock(mutex_); - cancelled_ = true; + switch (signature.null_policy) { + case KernelNullPolicy::AllValid: + case KernelNullPolicy::Nullable: + break; + default: + return kernel::FilterChunkRoute::Auto; } - ready_.notify_all(); - space_.notify_all(); - join(); } + return kernel::FilterChunkRoute::NativePredicate; +} - OperatorPtr child_; - // The operator this stage was built for. `profile_operator` wraps the - // STAGE, not the child, so the producer thread runs unwrapped code and had - // no profile frame at all: anything it submitted to the pool was attributed - // to no operator, and a fully parallel scan read as zero pool work. The - // producer pushes a scope for this entry so its work has an owner. - ExecutionProfileEntry* entry_ = nullptr; - std::thread producer_; - std::deque ready_chunks_; - std::mutex mutex_; - std::condition_variable ready_; - std::condition_variable space_; - std::optional failure_; - bool started_ = false; - bool producer_done_ = false; - bool cancelled_ = false; - bool done_ = false; -}; +auto build_filter_gather_map(const MapStep& step, OperatorPtr child, const ScalarRegistry* scalars, + const ExternRegistry* /*unused*/, const ExecutionContext& /*unused*/, + const std::vector* source_signature, + bool preserve_empty_morsels) + -> std::expected { + const auto& filter = ir::node_cast(*step.node); + return std::make_unique( + std::move(child), &filter.predicate(), scalars, + physical_filter_route(filter.predicate(), source_signature), preserve_empty_morsels); +} -[[nodiscard]] auto make_pipelined_stage(OperatorPtr child, const ExecutionContext& exec, - ExecutionProfileEntry* entry) -> OperatorPtr { - if (!exec.can_fan_out() || on_worker_pool_thread() || process_worker_pool().size() < 2) { - return child; - } - if (exec.parallel_stats != nullptr) { - exec.parallel_stats->pipelined_stages.fetch_add(1, std::memory_order_relaxed); +auto build_metadata_map(const MapStep& step, OperatorPtr child, const ScalarRegistry* /*unused*/, + const ExternRegistry* /*unused*/, const ExecutionContext& /*unused*/, + const std::vector* /*unused*/, bool /*unused*/) + -> std::expected { + if (step.node->kind() == ir::NodeKind::Project) { + const auto& project = ir::node_cast(*step.node); + return std::make_unique(std::move(child), &project.columns()); } - return std::make_unique(std::move(child), entry); + const auto& rename = ir::node_cast(*step.node); + return std::make_unique(std::move(child), &rename.renames()); } -[[nodiscard]] auto make_pipelined_stage_if(OperatorPtr child, bool eligible, - const ExecutionContext& exec, - ExecutionProfileEntry* entry) -> OperatorPtr { - return eligible ? make_pipelined_stage(std::move(child), exec, entry) : std::move(child); +auto build_row_local_update_map(const MapStep& step, OperatorPtr child, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, + const std::vector* /*unused*/, + bool /*unused*/) -> std::expected { + const auto& update = ir::node_cast(*step.node); + return std::make_unique(std::move(child), &update.fields(), scalars, + externs, exec); } -[[nodiscard]] auto scan_pipeline_worker_count(std::size_t unit_count) -> std::size_t { - auto& pool = process_worker_pool(); - if (pool.size() < 2) { - // With one pool thread there is no worker to reserve for a downstream - // operator batch. Running the producer there can deadlock as soon as a - // breaker submits work and waits, so keep the serial window source. - return 0; +auto build_filter_project_gather_map(const MapStep& step, OperatorPtr child, + const ScalarRegistry* scalars, + const ExternRegistry* /*unused*/, + const ExecutionContext& /*unused*/, + const std::vector* source_signature, + bool preserve_empty_morsels) + -> std::expected { + if (step.filter_predicate == nullptr || step.project_columns == nullptr) { + return std::unexpected("filter-project map step missing normalized operands"); } - // The decode pipeline is the one consumer sized against the POOL rather - // than the compute budget: it is what the extra threads were added for. - // `ExecutionContext::parallel_threads` deliberately does not cap it; - // configure_parallel_from_env uses that field for compute only. - const std::size_t budget = pool.size(); - std::size_t workers = std::min({budget, pool.size(), unit_count}); - // A spare thread is only necessary when every pool thread could remain - // parked behind ring backpressure. The ring holds 2W results and workers - // have already claimed at most another W units, so a source of at most 3W - // units necessarily lets one worker exit after the first chunk is released. - // Smaller sources (the common Parquet shape) keep the full decode budget; - // longer sources reserve one thread for downstream batches. - if (workers == pool.size() && unit_count > workers * 3) { - --workers; - } - return workers; + return std::make_unique( + std::move(child), step.filter_predicate, step.project_columns, scalars, + physical_filter_route(*step.filter_predicate, source_signature), preserve_empty_morsels); } -[[nodiscard]] auto build_pipelined_scan(const std::vector& operators, - bool count_as_pipeline, const DeferredScan& scan, - std::vector units, - const ScalarRegistry* scalars, - const ExternRegistry* externs, const ExecutionContext& exec) +auto build_filter_update_project_gather_map( + const MapStep& step, OperatorPtr child, const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec, + const std::vector* source_signature, bool preserve_empty_morsels) -> std::expected { - const std::size_t worker_count = scan_pipeline_worker_count(units.size()); - if (worker_count == 0) { - return std::unexpected("scan pipeline requires a worker"); + if (step.filter_predicate == nullptr || step.update_fields == nullptr || + step.project_columns == nullptr) { + return std::unexpected("filter-update-project map step missing normalized operands"); } - std::vector workers; - workers.reserve(worker_count); - for (std::size_t i = 0; i < worker_count; ++i) { - auto worker = build_scan_pipeline_worker(operators, scalars, externs, exec); - if (!worker.has_value()) { - return std::unexpected(std::move(worker.error())); - } - workers.push_back(std::move(*worker)); + robin_hood::unordered_set update_outputs; + robin_hood::unordered_set needed; + for (const auto& field : *step.update_fields) { + update_outputs.insert(field.alias); + collect_expr_column_refs(field.expr, needed); } - if (exec.parallel_stats != nullptr) { - if (count_as_pipeline) { - exec.parallel_stats->parallel_pipelines.fetch_add(1, std::memory_order_relaxed); + for (const auto& column : *step.project_columns) { + if (!update_outputs.contains(column.name)) { + needed.insert(column.name); } - exec.parallel_stats->morsels.fetch_add(units.size(), std::memory_order_relaxed); - exec.parallel_stats->pipelined_scans.fetch_add(1, std::memory_order_relaxed); } - return std::make_unique(scan, std::move(units), std::move(workers), exec, - process_worker_pool()); + std::vector gather_columns; + gather_columns.reserve(needed.size()); + for (const auto& name : needed) { + gather_columns.push_back(ir::ColumnRef{.name = name}); + } + return std::make_unique( + std::move(child), step.filter_predicate, step.update_fields, step.project_columns, + std::move(gather_columns), scalars, externs, exec, + physical_filter_route(*step.filter_predicate, source_signature), preserve_empty_morsels); } -/// A breaker only earns a scheduler thread when its probe input can actually -/// publish more than one source unit. Registered tables and one-unit readers -/// return a single chunk, so staging them merely moves the same serial call to -/// another thread. Keep this structural test at build time: it avoids putting -/// a speculative thread on the hot path and makes the queue capacity an -/// overlap buffer rather than an accidental materialization boundary. -[[nodiscard]] auto has_multi_unit_deferred_scan(const ir::Node& node, const TableRegistry& registry, - const ExecutionContext& exec) -> bool { - if (node.kind() == ir::NodeKind::Scan) { - const auto& scan = ir::node_cast(node); - if (registry.contains(scan.source_name())) { - return false; - } - const auto* deferred = exec.deferred_scan(scan.source_name()); - return deferred != nullptr && deferred->filter == nullptr && - deferred_scan_units(*deferred).size() > 1; - } - return std::ranges::any_of(node.children(), [&](const ir::NodePtr& child) { - return has_multi_unit_deferred_scan(*child, registry, exec); - }); +} // namespace + +auto map_kernel_factory(MapKernelCapability capability) noexcept -> MapKernelFactory { + static constexpr std::array factories{ + &build_filter_gather_map, + &build_metadata_map, + &build_row_local_update_map, + &build_filter_project_gather_map, + &build_filter_update_project_gather_map, + }; + const auto index = static_cast(capability); + return index < factories.size() ? factories[index] : nullptr; } +namespace physical_executor_detail { + /// Compose one step of a migrated physical map pipeline (Phase 1 of /// plans/kernel-pipeline-execution-plan.md). Walks the plan top-down so the /// per-step profile scopes nest exactly the way the per-kind switch's @@ -14613,7 +4470,8 @@ auto build_physical_map_step(const physical::Plan& plan, std::size_t index, if (exec.can_fan_out() && plan.mode == physical::PipelineMode::MorselParallel && index == plan.parallel_begin) { physical::note_map_pipeline_executed(); - return build_map_pipeline_parallel(plan, registry, scalars, externs, exec, model_out); + return pipeline_executor_detail::build_map_pipeline_parallel(plan, registry, scalars, + externs, exec, model_out); } const auto build_child = [&] -> std::expected { if (index + 1 == plan.steps.size()) { @@ -14649,18 +4507,18 @@ auto build_physical_map_step(const physical::Plan& plan, std::size_t index, } /// Build a join the plan migrated: `HashBuild` on one side, `HashProbe` on the -/// other, expressed for now as the streaming operators that already implement -/// exactly that. The kernel-pipeline plan's Phase 4 item 1. +/// other. The family-owned join executor implements both phases and exposes a +/// narrow construction boundary here. The kernel-pipeline plan's Phase 4 item +/// 1. /// /// The three branches are the ones that used to sit in `build_operator_impl`'s -/// per-kind switch, moved rather than rewritten -- which is the whole point of -/// this slice. Construction lives with the plan, the decisions are the same -/// ones `plan_join` already relayed, and the operators are untouched. The -/// backlog moves because a join is now executed *by* the physical plan, the -/// same sense in which a migrated map chain is: through a plan-owned builder -/// rather than the per-kind switch. What is still ahead is decomposing the -/// build and the probe into separate pipeline stages with a barrier between -/// them, so a probe can be a step inside a map pipeline. +/// per-kind switch. Construction lives with the plan, the decisions are the +/// same ones `plan_join` relayed, and execution is now owned by the join +/// family. HashBuild and HashProbe are explicit structural nodes connected by +/// a typed runtime-orientation edge; an eligible probe can also become a step +/// inside a map pipeline. + +namespace { /// Both of a streaming join's fan-out phases, resolved for this query. The /// capability halves are `physical::join_hash_build_parallelism` / @@ -14668,9 +4526,8 @@ auto build_physical_map_step(const physical::Plan& plan, std::size_t index, /// the `ExecutionContext` and the pool size, both in hand at build time. One /// definition, shared by every join construction site, the way `distinct_table` /// and `build_physical_distinct` share the dedup policy. -auto resolved_join_parallelism(const ExecutionContext& exec) -> physical::JoinParallelism { - physical::JoinParallelism par{.build = physical::join_hash_build_parallelism(), - .probe = physical::join_probe_parallelism()}; +auto resolve_join_parallelism(physical::JoinParallelism par, const ExecutionContext& exec) + -> physical::JoinParallelism { // The pool is sized for decode and spawns its threads on first touch, so a // serial query must not construct it just to learn it is serial. const std::size_t pool_size = exec.can_fan_out() ? process_worker_pool().size() : 0; @@ -14679,38 +4536,47 @@ auto resolved_join_parallelism(const ExecutionContext& exec) -> physical::JoinPa return par; } +/// Resolve the policies carried by the explicit HashBuild and HashProbe nodes. +/// Taking copies is intentional: resolution is execution-context state and the +/// data-only physical plan remains reusable and inspectable. +auto resolved_join_parallelism(const physical::StreamingJoinNodes& nodes, + const ExecutionContext& exec) -> physical::JoinParallelism { + return resolve_join_parallelism( + {.build = nodes.build.parallelism, .probe = nodes.probe.parallelism}, exec); +} + /// As `resolved_join_parallelism`, for the hash aggregate. The capability half -/// (floors, `min(budget, pool, 64)` ceiling, strategy) comes from -/// `aggregate_{partition,finalize}_parallelism`; the resolved half needs the +/// (floors, worker ceiling, strategy) comes from the four aggregate node +/// policy factories; the resolved half needs the /// `ExecutionContext` and pool size, both in hand here at build time. One /// definition, shared by every aggregate construction site. -auto resolved_aggregate_parallelism(const ExecutionContext& exec) - -> physical::AggregateParallelism { - physical::AggregateParallelism par{.partition = physical::aggregate_partition_parallelism(), - .finalize = physical::aggregate_finalize_parallelism()}; +auto resolved_aggregate_parallelism(const physical::HashAggregateNodes& nodes, + const ExecutionContext& exec) + -> std::expected { + physical::AggregateParallelism par{ + .discovery = nodes.discovery.parallelism, + .accumulation = nodes.accumulation.parallelism, + .final_ordering = nodes.final_ordering.parallelism, + .emission = nodes.emission.parallelism, + }; const std::size_t pool_size = exec.can_fan_out() ? process_worker_pool().size() : 0; - physical::resolve_breaker_parallelism(par.partition, exec, pool_size); - physical::resolve_breaker_parallelism(par.finalize, exec, pool_size); + physical::resolve_breaker_parallelism(par.discovery, exec, pool_size); + physical::resolve_breaker_parallelism(par.accumulation, exec, pool_size); + physical::resolve_breaker_parallelism(par.final_ordering, exec, pool_size); + physical::resolve_breaker_parallelism(par.emission, exec, pool_size); return par; } -/// Run a join's build phase here, at plan-execution time, instead of leaving it -/// to fire inside the probe's first `next()`. -/// -/// This is what makes the build *scheduled*: it has a caller that is not the -/// probe, and its completion is a point in the plan rather than a flag the probe -/// checks. `IBEX_JOIN_BUILD_LAZY=1` restores the old timing, which is both a kill -/// switch and the A/B handle for the one thing this changes that is not free -- -/// when the build side's child is drained. -auto scheduled_join_build(std::unique_ptr op) - -> std::expected { - static const bool lazy = std::getenv("IBEX_JOIN_BUILD_LAZY") != nullptr; - if (!lazy) { - if (auto err = op->run_build()) { - return std::unexpected(std::move(*err)); - } - } - return OperatorPtr{std::move(op)}; +} // namespace + +/// Compatibility construction sites that do not own a physical Plan still use +/// the same policy factories. Migrated joins take the `StreamingJoinNodes` +/// overload above instead. Exported via `physical_executor_internal.hpp` for +/// the pipeline unit's probe-fusion path. +auto resolved_join_parallelism(const ExecutionContext& exec) -> physical::JoinParallelism { + return resolve_join_parallelism({.build = physical::join_hash_build_parallelism(), + .probe = physical::join_probe_parallelism()}, + exec); } auto build_physical_join(const physical::Plan& plan, const ir::Node& node, @@ -14720,7 +4586,8 @@ auto build_physical_join(const physical::Plan& plan, const ir::Node& node, const auto& join = ir::node_cast(node); const physical::JoinPlan& jp = plan.join; if (jp.branch == physical::JoinBranch::SemiAnti) { - const bool stage_probe = has_multi_unit_deferred_scan(*join.children()[0], registry, exec); + const bool stage_probe = pipeline_executor_detail::has_multi_unit_deferred_scan( + *join.children()[0], registry, exec); // Multiple producers: tried the same overlap the inner-join site // once had here too, twice. First attempt (unbudgeted): q04 // regressed +19%. Second attempt, under a since-removed @@ -14742,11 +4609,18 @@ auto build_physical_join(const physical::Plan& plan, const ir::Node& node, if (!right.has_value()) { return std::unexpected(std::move(right.error())); } - return make_pipelined_stage_if(std::make_unique( - std::move(left_op.value()), std::move(right.value()), - join.kind(), &join.keys(), &exec), - stage_probe, exec, - execution_profile_entry(exec.execution_profile, node)); + return pipeline_executor_detail::make_pipelined_stage_if( + std::make_unique(std::move(left_op.value()), + std::move(right.value()), join.kind(), + &join.keys(), &exec), + stage_probe, exec, execution_profile_entry(exec.execution_profile, node)); + } + if (!plan.streaming_join.has_value()) { + return std::unexpected("physical join: streaming plan has no HashBuild/HashProbe nodes"); + } + const physical::StreamingJoinNodes& nodes = *plan.streaming_join; + if (auto edge_error = physical::validate_streaming_join_edge(nodes)) { + return std::unexpected(std::move(*edge_error)); } // `nulls equal` goes to the materialized join, which implements the // policy. These streaming operators hash and probe on their own and @@ -14755,7 +4629,8 @@ auto build_physical_join(const physical::Plan& plan, const ir::Node& node, // semantics -- and leaves this hot path bit-for-bit unchanged for every // join that does not ask for it. if (jp.branch == physical::JoinBranch::SingleKeyInner) { - const bool stage_probe = has_multi_unit_deferred_scan(*join.children()[0], registry, exec); + const bool stage_probe = pipeline_executor_detail::has_multi_unit_deferred_scan( + *join.children()[0], registry, exec); // A deferred probe scan must not be interpreted here — the join // publishes build-side bounds into its filter slot first, then // interprets the right subtree itself (resolve_deferred_probe). @@ -14774,29 +4649,31 @@ auto build_physical_join(const physical::Plan& plan, const ir::Node& node, return std::unexpected(std::move(left_op.error())); } if (probe.scan != nullptr) { - auto built = scheduled_join_build(std::make_unique( + auto built = make_scheduled_deferred_inner_join_operator( std::move(left_op.value()), join.children()[1].get(), ®istry, scalars, externs, exec, &join.keys(), probe.scan, *probe.name, join.suffix(), &join.pending_order(), - resolved_join_parallelism(exec))); + resolved_join_parallelism(nodes, exec), nodes.columns); if (!built.has_value()) { return std::unexpected(std::move(built.error())); } - return make_pipelined_stage_if(std::move(*built), stage_probe, exec, - execution_profile_entry(exec.execution_profile, node)); + return pipeline_executor_detail::make_pipelined_stage_if( + std::move(*built), stage_probe, exec, + execution_profile_entry(exec.execution_profile, node)); } auto right = materialize_row_local(*join.children()[1], registry, scalars, externs, exec, model_out); if (!right.has_value()) { return std::unexpected(std::move(right.error())); } - auto built = scheduled_join_build(std::make_unique( + auto built = make_scheduled_chunked_inner_join_operator( std::move(left_op.value()), std::move(right.value()), &join.keys(), exec, join.suffix(), - &join.pending_order(), resolved_join_parallelism(exec))); + &join.pending_order(), resolved_join_parallelism(nodes, exec), nodes.columns); if (!built.has_value()) { return std::unexpected(std::move(built.error())); } - return make_pipelined_stage_if(std::move(*built), stage_probe, exec, - execution_profile_entry(exec.execution_profile, node)); + return pipeline_executor_detail::make_pipelined_stage_if( + std::move(*built), stage_probe, exec, + execution_profile_entry(exec.execution_profile, node)); } // Streaming two-Int64-key inner join (plans/parallelism-overview.md's // "stream multi-key joins" item): same shape as the single-key @@ -14810,7 +4687,8 @@ auto build_physical_join(const physical::Plan& plan, const ir::Node& node, // `ChunkedInnerJoinOperator::resolve_deferred_probe_pair` for the // one-component filter this POC pushes into the scan. if (jp.branch == physical::JoinBranch::PairIntInner) { - const bool stage_probe = has_multi_unit_deferred_scan(*join.children()[0], registry, exec); + const bool stage_probe = pipeline_executor_detail::has_multi_unit_deferred_scan( + *join.children()[0], registry, exec); const auto probe = deferred_probe_scan_of(*join.children()[1], exec); auto left_op = build_operator(*join.children()[0], registry, scalars, externs, exec, model_out); @@ -14818,29 +4696,31 @@ auto build_physical_join(const physical::Plan& plan, const ir::Node& node, return std::unexpected(std::move(left_op.error())); } if (probe.scan != nullptr) { - auto built = scheduled_join_build(std::make_unique( + auto built = make_scheduled_deferred_inner_join_operator( std::move(left_op.value()), join.children()[1].get(), ®istry, scalars, externs, exec, &join.keys(), probe.scan, *probe.name, join.suffix(), &join.pending_order(), - resolved_join_parallelism(exec))); + resolved_join_parallelism(nodes, exec), nodes.columns); if (!built.has_value()) { return std::unexpected(std::move(built.error())); } - return make_pipelined_stage_if(std::move(*built), stage_probe, exec, - execution_profile_entry(exec.execution_profile, node)); + return pipeline_executor_detail::make_pipelined_stage_if( + std::move(*built), stage_probe, exec, + execution_profile_entry(exec.execution_profile, node)); } auto right = materialize_row_local(*join.children()[1], registry, scalars, externs, exec, model_out); if (!right.has_value()) { return std::unexpected(std::move(right.error())); } - auto built = scheduled_join_build(std::make_unique( + auto built = make_scheduled_chunked_inner_join_operator( std::move(left_op.value()), std::move(right.value()), &join.keys(), exec, join.suffix(), - &join.pending_order(), resolved_join_parallelism(exec))); + &join.pending_order(), resolved_join_parallelism(nodes, exec), nodes.columns); if (!built.has_value()) { return std::unexpected(std::move(built.error())); } - return make_pipelined_stage_if(std::move(*built), stage_probe, exec, - execution_profile_entry(exec.execution_profile, node)); + return pipeline_executor_detail::make_pipelined_stage_if( + std::move(*built), stage_probe, exec, + execution_profile_entry(exec.execution_profile, node)); } return std::unexpected("physical join: plan named no streaming branch"); @@ -14899,6 +4779,22 @@ auto build_physical_aggregate(const physical::Plan& plan, const ir::Node& node, return make_table_source(std::move(*result)); } if (ap.strategy == physical::AggregateStrategy::StreamingSorted) { + if (!plan.hash_aggregate.has_value()) { + return std::unexpected( + "physical aggregate: adaptive strategy has no hash-fallback phase chain"); + } + if (auto edge_error = physical::validate_hash_aggregate_edges(*plan.hash_aggregate)) { + return std::unexpected(std::move(*edge_error)); + } + if (agg.children().empty() || + plan.hash_aggregate->discovery.source != agg.children().front().get()) { + return std::unexpected( + "physical aggregate: Discovery input does not match the aggregate child"); + } + auto parallelism = resolved_aggregate_parallelism(*plan.hash_aggregate, exec); + if (!parallelism.has_value()) { + return std::unexpected(std::move(parallelism.error())); + } auto child_op = build_operator(*agg.children().front(), registry, scalars, externs, exec, model_out); if (!child_op.has_value()) { @@ -14906,20 +4802,19 @@ auto build_physical_aggregate(const physical::Plan& plan, const ir::Node& node, } // The sorted operator streams group-at-a-time when the child's // chunks arrive sorted on the group keys, and otherwise replays the - // first chunk into a hash ChunkedAggregateOperator — so it is safe + // first chunk into the hash aggregate phase operator — so it is safe // to route the whole streamable subset here. // Aggregates are often the terminal breaker and hash aggregation // emits only after consuming all input. Scheduling one in its own // stage in that shape buys no overlap and only creates a thread. // A join below it is staged instead: its probe stream can fill the // aggregate while it keeps pulling the next probe chunk. - // The plan describes the hash fallback's two fan-out phases; resolve - // them here, where the ExecutionContext is in hand, and hand them down. - // Observability only (slice 1): the operator still decides for itself - // and `check_agg_plan` aborts on disagreement. - return std::make_unique( - std::move(child_op.value()), &agg.group_by(), &agg.aggregations(), exec, - resolved_aggregate_parallelism(exec)); + // Resolve the hash fallback's four structural-node policies here, where + // the ExecutionContext is in hand, and hand them down. The operator + // retains only data-dependent gates such as actual row counts and + // strategy-specific usefulness thresholds. + return make_chunked_aggregate_operator(std::move(child_op.value()), &agg.group_by(), + &agg.aggregations(), exec, *parallelism, ap.columns); } return std::unexpected("physical aggregate: plan named no executable strategy"); @@ -15076,82 +4971,22 @@ auto build_physical_distinct(const physical::Plan& plan, const ir::Node& node, return std::make_unique(std::move(child_op.value()), dedup_plan); } +} // namespace physical_executor_detail + +namespace { + auto build_operator_impl(const ir::Node& node, const TableRegistry& registry, const ScalarRegistry* scalars, const ExternRegistry* externs, const ExecutionContext& exec, ModelResult* model_out) -> std::expected { - // Runtime-multithreading Phase 1 seam. Only consult the pipeline analysis - // when a parallel executor is actually requested — build_operator() is a - // hot query-construction path, so the serial default must not pay for - // analysis it would discard. When eligible, the whole row-local chain is - // built as one pipeline here and its inner nodes are not recursed into - // separately (only the pipeline's input subtree is), so there is no - // re-analysis of the chain and no infinite recursion. - // - // A lazy/deferred source in the pipeline's input subtree used to disqualify - // it, per the LazyTable synchronization contract's interim gate. That gate - // is LIFTED (Phase 3b): `build_morsel_pipeline` materializes its input - // subtree into an owned Table *before* constructing any morsel source, so - // every deferred decode happens on the single build thread and no worker - // ever reaches a `LazyTable`. The contract's hazards — concurrent `cache_` - // writes and concurrent `decode_` calls — need a worker to touch the source - // to arise, and none does. - // - // That is a claim about `build_morsel_pipeline`'s structure, so it is - // asserted there rather than restated here. A future slice that streams a - // source's morsels straight into workers, instead of materializing first, - // reintroduces both hazards and must re-establish eligibility (per-worker - // readers + a frozen cache) before it removes that assertion. // Physical-plan seam (plans/kernel-pipeline-execution-plan.md). One plan - // per node, and it describes the whole map chain: which steps run over - // morsels, what feeds them, and what runs serially above them. The - // composer walks it in both modes, so there is no arrangement of map nodes - // that only one of the two paths can express. + // per node, and it describes the whole map chain or migrated breaker. The + // executor below is also callable with an already-built plan, which makes + // plan-edge mutation tests exercise the same consumer production uses. const physical::Plan plan = physical::plan_physical(node, registry, externs); - if (plan.migrated && node.kind() == ir::NodeKind::Head) { - physical::note_map_pipeline_executed(); - return build_physical_head(node, registry, scalars, externs, exec, model_out); - } - if (plan.migrated && node.kind() == ir::NodeKind::Tail) { - physical::note_map_pipeline_executed(); - return build_physical_tail(node, registry, scalars, externs, exec, model_out); - } - if (plan.migrated && node.kind() == ir::NodeKind::TopK) { - physical::note_map_pipeline_executed(); - return build_physical_topk(node, registry, scalars, externs, exec, model_out); - } - if (plan.migrated && - (node.kind() == ir::NodeKind::FilterHead || node.kind() == ir::NodeKind::FilterTail)) { - physical::note_map_pipeline_executed(); - return build_physical_filter_head_tail(node, registry, scalars, externs, exec, model_out); - } - if (plan.migrated && node.kind() == ir::NodeKind::Distinct) { - physical::note_map_pipeline_executed(); - return build_physical_distinct(plan, node, registry, scalars, externs, exec, model_out); - } - if (plan.migrated && node.kind() == ir::NodeKind::Order) { - physical::note_map_pipeline_executed(); - return build_physical_order(node, registry, scalars, externs, exec, model_out); - } - if (plan.migrated && plan.aggregate.describes) { - physical::note_map_pipeline_executed(); - return build_physical_aggregate(plan, node, registry, scalars, externs, exec, model_out); - } - if (plan.migrated && plan.join.describes) { - physical::note_map_pipeline_executed(); - return build_physical_join(plan, node, registry, scalars, externs, exec, model_out); - } if (plan.migrated) { - // Every migrated plan, both modes: the composer walks the chain and - // hands the morsel run off at its boundary, and that run picks its own - // source strategy. Same constructors, same per-node profile entries, - // same source construction (an input the run does not stream goes - // through the public build_operator, so every Scan/ExternCall decision - // below is unchanged). - if (plan.mode != physical::PipelineMode::MorselParallel || !exec.can_fan_out()) { - physical::note_map_pipeline_executed(); - } - return build_physical_map_step(plan, 0, registry, scalars, externs, exec, model_out); + return build_migrated_physical_operator(plan, node, registry, scalars, externs, exec, + model_out); } // Counted in every mode. It used to fire only when the query could not fan // out, so at two cores or more the backlog read as empty -- a migration @@ -15177,12 +5012,13 @@ auto build_operator_impl(const ir::Node& node, const TableRegistry& registry, deferred != nullptr && deferred->filter == nullptr) { auto units = deferred_scan_units(*deferred); if (units.size() > 1) { - if (exec.can_fan_out() && scan_pipeline_worker_count(units.size()) > 0) { - return build_pipelined_scan({}, false, *deferred, std::move(units), scalars, - externs, exec); + if (exec.can_fan_out() && + pipeline_executor_detail::scan_pipeline_worker_count(units.size()) > 0) { + return pipeline_executor_detail::build_pipelined_scan( + {}, false, *deferred, std::move(units), scalars, externs, exec); } - return std::make_unique(*deferred, std::move(units), - exec); + return pipeline_executor_detail::make_deferred_scan_source( + *deferred, std::move(units), exec); } } } @@ -15198,8 +5034,8 @@ auto build_operator_impl(const ir::Node& node, const TableRegistry& registry, if (!child_op.has_value()) { return std::unexpected(std::move(child_op.error())); } - return build_row_local_map_operator(node, std::move(child_op.value()), scalars, externs, - exec, false); + return pipeline_executor_detail::build_row_local_map_operator( + node, std::move(child_op.value()), scalars, externs, exec, false); } if (node.kind() == ir::NodeKind::Project) { @@ -15212,8 +5048,8 @@ auto build_operator_impl(const ir::Node& node, const TableRegistry& registry, if (!child_op.has_value()) { return std::unexpected(std::move(child_op.error())); } - return build_row_local_map_operator(node, std::move(child_op.value()), scalars, externs, - exec, false); + return pipeline_executor_detail::build_row_local_map_operator( + node, std::move(child_op.value()), scalars, externs, exec, false); } // No FilterHead / FilterTail branch: fused Head(Filter(x)) / Tail(Filter(x)) @@ -15230,8 +5066,8 @@ auto build_operator_impl(const ir::Node& node, const TableRegistry& registry, if (!child_op.has_value()) { return std::unexpected(std::move(child_op.error())); } - return build_row_local_map_operator(node, std::move(child_op.value()), scalars, externs, - exec, false); + return pipeline_executor_detail::build_row_local_map_operator( + node, std::move(child_op.value()), scalars, externs, exec, false); } if (node.kind() == ir::NodeKind::ExternCall && externs != nullptr) { @@ -15278,349 +5114,18 @@ auto build_operator_impl(const ir::Node& node, const TableRegistry& registry, // TopK stays a serial bounded-heap select (O(n log k)); Tail materializes // and calls `tail_table`; the plan just records that they are breakers. - if (node.kind() == ir::NodeKind::Columns) { - if (node.children().empty()) { - return std::unexpected("columns node missing child"); - } - return build_unary_materializing_operator(*node.children().front(), registry, scalars, - externs, exec, model_out, - [](Table input) { return columns_table(input); }); - } - - if (node.kind() == ir::NodeKind::Melt) { - const auto& mn = ir::node_cast(node); - if (mn.children().empty()) { - return std::unexpected("melt node missing child"); - } - return build_unary_materializing_operator( - *mn.children().front(), registry, scalars, externs, exec, model_out, - [&](Table input) { return melt_table(input, mn.id_columns(), mn.measure_columns()); }); - } - - if (node.kind() == ir::NodeKind::Dcast) { - const auto& dn = ir::node_cast(node); - if (dn.children().empty()) { - return std::unexpected("dcast node missing child"); - } - return build_unary_materializing_operator( - *dn.children().front(), registry, scalars, externs, exec, model_out, [&](Table input) { - return dcast_table(input, dn.pivot_column(), dn.value_column(), dn.row_keys()); - }); - } - - if (node.kind() == ir::NodeKind::Cov) { - if (node.children().empty()) { - return std::unexpected("cov node missing child"); - } - return build_unary_materializing_operator(*node.children().front(), registry, scalars, - externs, exec, model_out, - [](Table input) { return cov_table(input); }); - } - - if (node.kind() == ir::NodeKind::Corr) { - if (node.children().empty()) { - return std::unexpected("corr node missing child"); - } - return build_unary_materializing_operator(*node.children().front(), registry, scalars, - externs, exec, model_out, - [](Table input) { return corr_table(input); }); - } - - if (node.kind() == ir::NodeKind::Transpose) { - if (node.children().empty()) { - return std::unexpected("transpose node missing child"); - } - return build_unary_materializing_operator( - *node.children().front(), registry, scalars, externs, exec, model_out, - [](Table input) { return transpose_table(input); }); - } - - if (node.kind() == ir::NodeKind::Join) { - const auto& join = ir::node_cast(node); - if (join.children().size() != 2) { - return std::unexpected("join node expects exactly two children"); - } - // Only the materializing join reaches here now. A streaming one is a - // migrated plan and was built by `build_physical_join` at the seam - // above, the same way a migrated map chain never reaches this switch. - const ir::Expr* pred = join.predicate().has_value() ? &*join.predicate() : nullptr; - return build_binary_materializing_operator( - *join.children()[0], *join.children()[1], registry, scalars, externs, exec, model_out, - [&](Table left, Table right) { - return join_table_impl(left, right, join.kind(), join.keys(), pred, scalars, - compute_mask, join.suffix(), join.pending_order(), - join.null_match(), join.expect(), join.take(), &exec); - }); - } - - if (node.kind() == ir::NodeKind::Matmul) { - if (node.children().size() != 2) { - return std::unexpected("matmul node expects exactly two children"); - } - return build_binary_materializing_operator( - *node.children()[0], *node.children()[1], registry, scalars, externs, exec, model_out, - [](Table left, Table right) { return matmul_table(left, right); }); - } - - if (node.kind() == ir::NodeKind::Update) { - const auto& update = ir::node_cast(node); - if (update.children().empty()) { - return std::unexpected("update node missing child"); - } - if (update.guard() != nullptr) { - return build_unary_materializing_operator( - *update.children().front(), registry, scalars, externs, exec, model_out, - [&](Table input) -> std::expected { - return apply_guarded_update(std::move(input), update, scalars, externs, exec); - }); - } - if (!update.group_by().empty()) { - const bool all_rank = std::all_of( - update.fields().begin(), update.fields().end(), [](const ir::FieldSpec& f) { - return std::holds_alternative(f.expr.node); - }); - if (!all_rank && update.tuple_fields().empty()) { - return build_unary_materializing_operator( - *update.children().front(), registry, scalars, externs, exec, model_out, - [&](Table input) -> std::expected { - return grouped_update_table(std::move(input), update.fields(), - update.group_by(), scalars, externs, exec); - }); - } - if (!all_rank || !update.tuple_fields().empty()) { - return std::unexpected( - "update + by: tuple-bound fields are not yet supported in grouped updates"); - } - return build_unary_materializing_operator( - *update.children().front(), registry, scalars, externs, exec, model_out, - [&](Table input) -> std::expected { - Table result = std::move(input); - for (const auto& field : update.fields()) { - const auto* rank = std::get_if(&field.expr.node); - auto res = evaluate_rank_column(result, *rank, update.group_by(), exec); - if (!res) { - return std::unexpected(res.error()); - } - if (res->validity.has_value()) { - result.add_column(field.alias, std::move(res->column), - std::move(*res->validity)); - } else { - result.add_column(field.alias, std::move(res->column)); - } - } - return std::expected{std::move(result)}; - }); - } - // Route to a streaming ChunkedUpdateOperator when every field is - // row-local and there are no table-valued tuple assignments. - const bool all_row_local = - std::all_of(update.fields().begin(), update.fields().end(), - [](const ir::FieldSpec& f) { return is_row_local_update_expr(f.expr); }); - if (all_row_local && update.tuple_fields().empty()) { - auto child_op = build_operator(*update.children().front(), registry, scalars, externs, - exec, model_out); - if (!child_op.has_value()) { - return std::unexpected(std::move(child_op.error())); - } - return std::make_unique( - std::move(child_op.value()), &update.fields(), scalars, externs, exec); - } - auto child = build_unary_materializing_operator( - *update.children().front(), registry, scalars, externs, exec, model_out, - [&](Table input) { - return update_table(std::move(input), update.fields(), scalars, externs, exec); - }); - if (!child.has_value()) { - return std::unexpected(std::move(child.error())); - } - auto result = materialize_operator(std::move(child.value())); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - for (const auto& tspec : update.tuple_fields()) { - auto src = interpret_node(*tspec.source, registry, scalars, externs, exec); - if (!src.has_value()) { - return std::unexpected(std::move(src.error())); - } - if (tspec.aliases.empty()) { - for (const auto& entry : src->columns) { - if (entry.validity) { - result->add_column(entry.name, *entry.column, *entry.validity); - } else { - result->add_column(entry.name, *entry.column); - } - } - } else { - if (src->columns.size() != tspec.aliases.size()) { - return std::unexpected( - "tuple assignment: expected " + std::to_string(tspec.aliases.size()) + - " column(s), got " + std::to_string(src->columns.size())); - } - for (std::size_t i = 0; i < tspec.aliases.size(); ++i) { - const auto& entry = src->columns[i]; - if (entry.validity) { - result->add_column(tspec.aliases[i], *entry.column, *entry.validity); - } else { - result->add_column(tspec.aliases[i], *entry.column); - } - } - } - } - return make_table_source(std::move(result.value())); - } - - if (node.kind() == ir::NodeKind::Resample) { - const auto& rs = ir::node_cast(node); - if (node.children().empty()) { - return std::unexpected("resample node missing child"); - } - return build_unary_materializing_operator( - *node.children().front(), registry, scalars, externs, exec, model_out, - [&](Table input) { - return resample_table(input, rs.duration(), rs.group_by(), rs.aggregations()); - }); - } - - if (node.kind() == ir::NodeKind::Window) { - const auto& win = ir::node_cast(node); - if (node.children().empty()) { - return std::unexpected("window node missing child"); - } - const ir::Node& child_node = *node.children().front(); - if (child_node.kind() != ir::NodeKind::Update) { - return std::unexpected( - "window: only 'update' is currently supported inside a window block"); - } - const auto& update_node = ir::node_cast(child_node); - if (child_node.children().empty()) { - return std::unexpected("window: update node missing child"); - } - auto source_op = build_operator(*child_node.children().front(), registry, scalars, externs, - exec, model_out); - if (!source_op.has_value()) { - return std::unexpected(std::move(source_op.error())); - } - auto source = materialize_operator(std::move(source_op.value())); - if (!source.has_value()) { - return std::unexpected(std::move(source.error())); - } - if (!source->time_index().has_value()) { - return std::unexpected( - "window requires a TimeFrame — use as_timeframe() to designate a timestamp column"); - } - auto result = - update_node.group_by().empty() - ? windowed_update_table(std::move(source.value()), update_node.fields(), - win.duration(), scalars, externs, exec, win.aligned()) - : grouped_windowed_update_table(std::move(source.value()), update_node.fields(), - win.duration(), update_node.group_by(), scalars, - externs, exec, win.aligned()); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - if (win.select_only()) { - // `window` + `select`: keep only the time index, group keys, and the - // listed fields (time index first so the result stays a TimeFrame). - std::vector keep; - robin_hood::unordered_set seen; - auto keep_col = [&](const std::string& name) { - if (seen.insert(name).second) { - keep.push_back(ir::ColumnRef{.name = name}); - } - }; - if (result->time_index().has_value()) { - keep_col(*result->time_index()); - } - for (const auto& key : update_node.group_by()) { - keep_col(key.name); - } - for (const auto& field : update_node.fields()) { - keep_col(field.alias); - } - // A grouped window leaves the rows group-major, and `project_table` - // preserves that: it derives with `RowTransform::Preserve`, which - // carries `grouped_by` through and drops the ordering only if the - // projection removes one of its keys, and the TimeFrame invariant - // leaves a group-major ordering alone rather than rewriting it to - // the (false) "time index ascending". - auto projected = project_table(result.value(), keep); - if (!projected.has_value()) { - return std::unexpected(std::move(projected.error())); - } - result = std::move(projected); - } - return make_table_source(std::move(result.value())); - } - - if (node.kind() == ir::NodeKind::AsTimeframe) { - const auto& atf = ir::node_cast(node); - if (node.children().empty()) { - return std::unexpected("as_timeframe node missing child"); - } - auto child_op = - build_operator(*node.children().front(), registry, scalars, externs, exec, model_out); - if (!child_op.has_value()) { - return std::unexpected(std::move(child_op.error())); - } - return std::make_unique(std::move(child_op.value()), - atf.column(), exec); - } - - if (node.kind() == ir::NodeKind::Model) { - const auto& mn = ir::node_cast(node); - if (mn.children().empty()) { - return std::unexpected("model node missing child"); - } - auto child_op = - build_operator(*mn.children().front(), registry, scalars, externs, exec, model_out); - if (!child_op.has_value()) { - return std::unexpected(std::move(child_op.error())); - } - auto input = materialize_operator(std::move(child_op.value())); - if (!input.has_value()) { - return std::unexpected(std::move(input.error())); - } - auto result = - fit_model(input.value(), mn.formula(), mn.method(), mn.params(), scalars, externs); - if (!result.has_value()) { - return std::unexpected(std::move(result.error())); - } - // Linear methods expose coefficients; tree models expose importance; - // unsupervised models (e.g. kmeans) have neither, so fall back to the - // per-row fitted output (e.g. cluster ids). - Table primary = !result.value().coefficients.columns.empty() ? result.value().coefficients - : !result.value().importance.columns.empty() ? result.value().importance - : result.value().fitted_values; - if (model_out != nullptr) { - *model_out = std::move(result.value()); - } - return make_table_source(std::move(primary)); - } - - if (node.kind() == ir::NodeKind::Construct || node.kind() == ir::NodeKind::Stream) { - auto table = interpret_node(node, registry, scalars, externs, exec, model_out); - if (!table.has_value()) { - return std::unexpected(std::move(table.error())); - } - return make_table_source(std::move(table.value())); - } - - if (node.kind() == ir::NodeKind::Program) { - const auto& program = ir::node_cast(node); - auto preamble = execute_program_preamble(program.preamble(), scalars, externs); - if (!preamble.has_value()) { - return std::unexpected(std::move(preamble.error())); - } - return build_operator(program.main_node(), registry, scalars, externs, exec, model_out); - } - - // Remaining node kinds fall through to interpret_node. Scan is already - // handled as a source by the caller. - auto table = interpret_node(node, registry, scalars, externs, exec, model_out); - if (!table.has_value()) { - return std::unexpected(std::move(table.error())); - } - return make_table_source(std::move(table.value())); + // Every other node kind is a materialized-call fallback: not migrated by + // `plan_physical`, counted just above, and executed by `interpret_node` + // rather than a per-kind branch. `build_materialized_fallback` still builds + // the breaker's direct children through the physical path (fused parallel + // scan, streaming join), so the switch's 15 hand-synced branches collapse + // to one without regressing a filtered/projected input. Construct / Stream / + // Program (preamble), Model (`model_out`), and the reshape / stat / window / + // update / matmul / materializing-join kinds all resolve there. + // `physical_fallbacks_for(kind)` buckets the backlog so a kind can later be + // lifted to a migrated breaker-over-pipeline the way Join / Aggregate / + // Order were. Scan is handled as a source by the caller. + return build_materialized_fallback(node, registry, scalars, externs, exec, model_out); } } // namespace diff --git a/src/runtime/interpreter.cpp b/src/runtime/interpreter.cpp index 4efe3220..ad872f63 100644 --- a/src/runtime/interpreter.cpp +++ b/src/runtime/interpreter.cpp @@ -374,6 +374,19 @@ auto interpret_node(const ir::Node& node, const TableRegistry& registry, if (interrupt_requested()) { return std::unexpected(interrupt_message()); } + // The materialized-call fallback pre-builds a breaker's direct children + // through the physical path so a filtered/projected input keeps its fused + // parallel scan. Hand back the pre-built table rather than re-evaluating the + // subtree whole-table and serial. The list holds only direct children, so + // this never short-circuits the fallback node itself; a linear scan is + // right for a list that is at most binary. + if (exec.pre_materialized_children != nullptr) { + for (const auto& [child, table] : *exec.pre_materialized_children) { + if (child == &node) { + return *table; + } + } + } switch (node.kind()) { case ir::NodeKind::Scan: { const auto& scan = ir::node_cast(node); diff --git a/src/runtime/interpreter_internal.hpp b/src/runtime/interpreter_internal.hpp index 4ffb94c2..e747b481 100644 --- a/src/runtime/interpreter_internal.hpp +++ b/src/runtime/interpreter_internal.hpp @@ -43,6 +43,10 @@ namespace ibex::runtime { +namespace physical { +struct Plan; +} + // `invariant_violation` and the gather kernel (`make_gather_column`, // `gather_range_into`, `gather_validity_range`) moved down to // runtime_internal.hpp, which this includes: `gather_column` lives in that @@ -1751,6 +1755,14 @@ enum class FloatCleanMode : std::uint8_t { const ScalarRegistry* scalars, const ExternRegistry* externs, const ExecutionContext& exec, ModelResult* model_out) -> std::expected; +/// Execute an already-built migrated physical plan. Production's normal +/// `build_operator` path calls the same implementation after planning; keeping +/// this seam explicit lets mutation tests prove that executor behavior follows +/// the plan rather than a reconstructed policy. +[[nodiscard]] auto build_operator_from_physical_plan( + const physical::Plan& plan, const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, const ExecutionContext& exec, + ModelResult* model_out) -> std::expected; [[nodiscard]] auto materialize_operator(OperatorPtr op) -> std::expected; [[nodiscard]] auto evaluate_rank_column(const Table& input, const ir::RankExpr& rank, const std::vector& group_by, diff --git a/src/runtime/join_chunked.cpp b/src/runtime/join_chunked.cpp new file mode 100644 index 00000000..6b988e40 --- /dev/null +++ b/src/runtime/join_chunked.cpp @@ -0,0 +1,2872 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +// chunked.cpp — streaming (chunked) operator pipeline: per-chunk operators, +// rank evaluation, extern-call execution, and build_operator plan construction. +// Split out of interpreter.cpp; shared declarations live in interpreter_internal.hpp. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "physical_plan.hpp" + +#if defined(__AVX2__) || defined(__BMI2__) +#include +#endif + +#include "chunk_conversion_internal.hpp" +#include "execution_profile_internal.hpp" +#include "interpreter_internal.hpp" +#include "join_chunked_internal.hpp" +#include "join_internal.hpp" +#include "kernel_filter.hpp" +#include "kernel_types.hpp" +#include "kernel_update.hpp" +#include "model_internal.hpp" +#include "reshape_internal.hpp" +#include "runtime_internal.hpp" + +namespace ibex::runtime { + +namespace { + +/// The base Scan under `node`, peeled through a chain of Project/Rename/Update +/// wrappers — null when `node` is not (a simple wrapper around) one scan. +/// Deliberately not past Filter: `deferred_probe_scan_of` reuses this peel, +/// and the driver only ever registers a probe scan for exactly the +/// Project/Rename/Update shape it proved eligible, so widening the peel here +/// would silently widen what counts as a deferred probe too. +auto base_scan_of(const ir::Node& node) -> const ir::ScanNode* { + const ir::Node* cur = &node; + while (cur->kind() == ir::NodeKind::Project || cur->kind() == ir::NodeKind::Rename || + cur->kind() == ir::NodeKind::Update) { + if (cur->children().size() != 1 || cur->children().front() == nullptr) { + return nullptr; + } + cur = cur->children().front().get(); + } + if (cur->kind() != ir::NodeKind::Scan) { + return nullptr; + } + return &ir::node_cast(*cur); +} + +/// If `right` is a chain of Project/Rename nodes over a Scan whose name the +/// driver registered as a deferred probe scan, return its registration. The +/// driver only registers scans it proved eligible (ir::deferrable_probe_scans: +/// occurs once, feeds exactly this shape), so a hit here IS the eligible +/// position. +auto deferred_probe_scan_impl(const ir::Node& right, const ExecutionContext& exec) + -> DeferredProbeScan { + if (exec.deferred_scans == nullptr) { + return {}; + } + const auto* scan_node = base_scan_of(right); + if (scan_node == nullptr) { + return {}; + } + const auto& name = scan_node->source_name(); + const auto* scan = exec.deferred_scan(name); + // A probe scan is one with a filter slot to publish build-side bounds + // into. The registry also holds streaming registrations (Phase 1), which + // have no slot and are not this join's to decode. + if (scan == nullptr || scan->filter == nullptr) { + return {}; + } + // Recover the stored key iterator to expose the registry's own name string. + const auto it = exec.deferred_scans->find(name); + return DeferredProbeScan{.scan = scan, .name = &it->first}; +} + +/// The end of a hash chain. At namespace scope because the index and the +/// operator that probes it are no longer the same type, so the sentinel belongs +/// to neither alone; `ChunkedInnerJoinOperator::kNil` aliases it. +inline constexpr std::size_t kJoinNil = std::numeric_limits::max(); + +/// A hash-index head table split into partitions by key hash. +/// +/// Every key belongs to exactly one partition, so P workers can fill P +/// partitions with no shared writes, no locks, and -- unlike per-worker maps -- +/// no merge afterwards. That is what makes a hash build morsel-parallel, and +/// it is the reason this type exists: `build_join_hash_index` is one serial +/// loop, and on TPC-H q21 it spends 40 ms hashing 1.29M rows inside a 75 ms +/// query (measured 2026-08-25, see plans/kernel-pipeline-execution-plan.md, +/// "Where join time actually goes"). +/// +/// `partition_count == 1` is exactly the single-map behaviour this replaced, +/// bit for bit: one partition, mask 0, every key landing in `parts[0]`. +/// Partitioning the TYPE and filling it in parallel are deliberately separate +/// steps -- the first cannot change a result, so anything the second breaks is +/// unambiguously the second's fault. +template , class Eq = std::equal_to> +struct PartitionedHeads { + using Map = robin_hood::unordered_flat_map; + /// Always a power of two, so `part_of` is a mask rather than a modulo. + std::vector parts{1}; + std::size_t mask = 0; + + /// Size to `count` partitions (rounded down to a power of two, at least 1). + void partition(std::size_t count) { + std::size_t p = 1; + while (p * 2 <= count) { + p *= 2; + } + parts.assign(p, Map{}); + mask = p - 1; + } + + [[nodiscard]] auto partition_count() const noexcept -> std::size_t { return parts.size(); } + + [[nodiscard]] auto part_of(const Key& key) const noexcept -> std::size_t { + return mask == 0 ? 0 : (Hash{}(key)&mask); + } + + /// Reserve for `n` build rows. Split across partitions, since a key can + /// only land in one of them. + void reserve(std::size_t n) { + const std::size_t per = (n / parts.size()) + 1; + for (auto& part : parts) { + part.reserve(per); + } + } + + /// Insert `row` as the head for `key` if absent. Returns a pointer to the + /// stored head (never null) and whether it was newly inserted, so a caller + /// that loses the race to an earlier row can chain onto what is there. + auto try_emplace(const Key& key, std::size_t row) -> std::pair { + auto [it, inserted] = parts[part_of(key)].try_emplace(key, row); + return {&it->second, inserted}; + } + + /// The head row for `key`, or `kJoinNil` when the build side has none. + [[nodiscard]] auto find_head(const Key& key) const -> std::size_t { + const auto& part = parts[part_of(key)]; + const auto it = part.find(key); + return it == part.end() ? kJoinNil : it->second; + } +}; + +/// Everything a hash build produces and a hash probe consumes: the chained +/// index over one side's key column, plus what the probe needs to interpret it. +/// +/// This is the barrier between Phase 4's `HashBuild` and `HashProbe`, stated as +/// a type. A probe holds it as `shared_ptr`: it cannot +/// write to what a build produced, and one build can feed several probes. +/// +/// What is deliberately NOT here: the categorical code -> head table. It is +/// derived from the PROBE chunk's dictionary and rebuilt per chunk, so it is +/// the probing operator's state (`probe_code_heads_`). Holding it here is what +/// made the build state mutable during probing. +struct JoinHashIndex { + ExprType key_kind = ExprType::Int; + /// False once any key repeats: the probe can skip chain walking when a + /// build side is unique. + bool unique = true; + /// Row -> next row with the same key, `kJoinNil` at the end of a chain. + std::vector chain_next; + /// Head row per key, one map per key representation. + PartitionedHeads i64_heads; + PartitionedHeads f64_heads; + PartitionedHeads bool_heads; + PartitionedHeads date_heads; + PartitionedHeads ts_heads; + PartitionedHeads string_heads; + + /// Two-fixed-width-int-key path: both key values pack into one struct, + /// injective with no knowledge of their domains -- same trick as the + /// aggregate's own `PairIntKey`. + struct PairKey { + std::uint64_t a = 0; + std::uint64_t b = 0; + [[nodiscard]] friend auto operator==(const PairKey&, const PairKey&) -> bool = default; + }; + struct PairKeyHash { + auto operator()(const PairKey& key) const noexcept -> std::size_t { + std::uint64_t h = key.a * 0x9e3779b97f4a7c15ULL; + h ^= key.b + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2); + return static_cast(h); + } + }; + PartitionedHeads pair_heads; + + /// Borrowed from the build table; null when the key column has no nulls. A + /// null key matches nothing, so null build rows are never indexed and null + /// probe rows are never looked up. + const ValidityBitmap* validity = nullptr; +}; + +auto detect_join_key_kind(const ColumnValue& col, ExprType& out) -> std::optional { + if (std::holds_alternative>(col)) { + out = ExprType::Int; + } else if (std::holds_alternative>(col)) { + out = ExprType::Double; + } else if (std::holds_alternative>(col)) { + out = ExprType::Bool; + } else if (std::holds_alternative>(col)) { + out = ExprType::Date; + } else if (std::holds_alternative>(col)) { + out = ExprType::Timestamp; + } else if (std::holds_alternative>(col) || + std::holds_alternative>(col)) { + out = ExprType::String; + } else { + return "ChunkedInnerJoinOperator: unsupported key type"; + } + return std::nullopt; +} + +/// The hash build. Chains each build row to the next row carrying the same key, +/// iterating in reverse so a chain walks forward during the probe -- which is +/// what makes the streamed output match the nested-loop inner join's ordering. +/// +/// A null key matches nothing, not even another null (SQL / Polars). So a +/// null-keyed build row is never indexed, and a null-keyed probe row is never +/// looked up. Both halves are needed: a null cell holds the type's zero value, +/// so a null probe key would otherwise find a genuine `0`. +/// Fill one `PartitionedHeads` from `n` build rows, serially when it has one +/// partition and over the worker pool when it has more. +/// +/// The two paths must produce a bit-identical index, and the reason they do is +/// worth stating rather than trusting. The serial build walks rows from `n-1` +/// down to 0, so a key's head is its LOWEST row and its chain ascends. The +/// parallel build scatters rows into partitions keeping them ascending within +/// each, then walks each partition's slice in reverse -- and because every row +/// carrying a given key lands in that key's one partition, reverse order +/// within the partition is reverse order within the key. Same head, same +/// chain, in any partition count. Output row order is a join contract +/// (SPEC.md 5.6 leaves it open, but `emit_swapped` and the chained probe both +/// depend on the chain's direction), so this is not a detail. +/// +/// Writes are disjoint by construction: `chain_next[row]` is written only by +/// the worker owning that row's partition, and each partition's map is its +/// own. No locks, and no merge afterwards -- which is the whole reason the +/// head table is partitioned rather than built per-worker and combined. +template +void fill_partitioned_heads(Heads& heads, std::size_t n, const KeyAt& key_at, const IsNull& is_null, + std::vector& chain_next, bool& unique) { + const auto insert = [&](std::size_t row, bool& dup) { + auto [head, inserted] = heads.try_emplace(key_at(row), row); + if (!inserted) { + chain_next[row] = *head; + *head = row; + dup = true; + } + }; + if (heads.partition_count() == 1) { + heads.reserve(n); + bool dup = false; + for (std::size_t r = n; r-- > 0;) { + if (is_null(r)) { + continue; + } + insert(r, dup); + } + unique = unique && !dup; + return; + } + + const std::size_t parts = heads.partition_count(); + auto& pool = process_worker_pool(); + const std::size_t ranges = std::min(pool.size(), parts); + const std::size_t grain = (n + ranges - 1) / ranges; + heads.reserve(n); + + // Pass 1: which partition each row belongs to, and how many rows each + // (range, partition) pair contributes. Null-keyed rows are never indexed, + // so they are given no partition and never counted. + std::vector part_of_row(n, 0); + std::vector indexed(n, 0); + std::vector counts(ranges * parts, 0); + { + auto batch = pool.submit(ranges, [&](std::size_t r) { + const std::size_t begin = r * grain; + const std::size_t end = std::min(n, begin + grain); + std::size_t* row_counts = counts.data() + (r * parts); + for (std::size_t row = begin; row < end; ++row) { + if (is_null(row)) { + continue; + } + const std::size_t part = heads.part_of(key_at(row)); + part_of_row[row] = static_cast(part); + indexed[row] = 1; + ++row_counts[part]; + } + }); + batch.wait(); + } + + std::vector offsets(ranges * parts, 0); + std::vector part_begin(parts + 1, 0); + { + std::size_t running = 0; + for (std::size_t p = 0; p < parts; ++p) { + part_begin[p] = running; + for (std::size_t r = 0; r < ranges; ++r) { + offsets[(r * parts) + p] = running; + running += counts[(r * parts) + p]; + } + } + part_begin[parts] = running; + } + + // Pass 2: scatter. Ranges are laid out in ascending order within each + // partition and each range walks ascending, so a partition's slice is + // ascending in row index -- which is what pass 3 reverses. + std::vector scatter_rows(part_begin[parts]); + { + auto batch = pool.submit(ranges, [&](std::size_t r) { + const std::size_t begin = r * grain; + const std::size_t end = std::min(n, begin + grain); + std::size_t* cursor = offsets.data() + (r * parts); + for (std::size_t row = begin; row < end; ++row) { + if (indexed[row] == 0) { + continue; + } + scatter_rows[cursor[part_of_row[row]]++] = row; + } + }); + batch.wait(); + } + + // Pass 3: one worker per partition, claimed dynamically. + std::vector dup(parts, 0); + { + std::atomic cursor{0}; + auto batch = pool.submit(std::min(pool.size(), parts), [&](std::size_t) { + for (std::size_t p = cursor.fetch_add(1, std::memory_order_relaxed); p < parts; + p = cursor.fetch_add(1, std::memory_order_relaxed)) { + bool part_dup = false; + for (std::size_t i = part_begin[p + 1]; i-- > part_begin[p];) { + insert(scatter_rows[i], part_dup); + } + dup[p] = part_dup ? 1 : 0; + } + }); + batch.wait(); + } + for (const char d : dup) { + unique = unique && d == 0; + } +} + +auto build_join_hash_index(const Table& build_side, const std::string& key_name, ExprType key_kind, + std::size_t partitions) -> std::expected { + const ColumnValue* key = build_side.find(key_name); + if (key == nullptr) { + return std::unexpected("join key not found in build side: " + key_name); + } + JoinHashIndex index; + index.key_kind = key_kind; + const auto* build_entry = build_side.find_entry(key_name); + index.validity = build_entry != nullptr && build_entry->validity.has_value() + ? &*build_entry->validity + : nullptr; + const std::size_t n = build_side.rows(); + index.chain_next.assign(n, kJoinNil); + + const auto is_null = [&index](std::size_t row) noexcept { + return index.validity != nullptr && !(*index.validity)[row]; + }; + const auto build_scalar = [&](const ColT& col, Heads& heads) { + const auto* data = col.data(); + heads.partition(partitions); + fill_partitioned_heads( + heads, col.size(), [data](std::size_t r) { return data[r]; }, is_null, index.chain_next, + index.unique); + }; + + if (key_kind == ExprType::Int) { + const auto* col = std::get_if>(key); + if (col == nullptr) + return std::unexpected("inner join: build-side key type mismatch"); + build_scalar(*col, index.i64_heads); + } else if (key_kind == ExprType::Double) { + const auto* col = std::get_if>(key); + if (col == nullptr) + return std::unexpected("inner join: build-side key type mismatch"); + build_scalar(*col, index.f64_heads); + } else if (key_kind == ExprType::Bool) { + const auto* col = std::get_if>(key); + if (col == nullptr) + return std::unexpected("inner join: build-side key type mismatch"); + // No `data()` on a packed bool column, so this is the one + // representation `build_scalar` cannot serve. Left unpartitioned: a + // bool key has two values, so partitioning can only leave every + // partition but two empty. + fill_partitioned_heads( + index.bool_heads, n, [col](std::size_t r) { return (*col)[r]; }, is_null, + index.chain_next, index.unique); + } else if (key_kind == ExprType::Date) { + const auto* col = std::get_if>(key); + if (col == nullptr) + return std::unexpected("inner join: build-side key type mismatch"); + build_scalar(*col, index.date_heads); + } else if (key_kind == ExprType::Timestamp) { + const auto* col = std::get_if>(key); + if (col == nullptr) + return std::unexpected("inner join: build-side key type mismatch"); + build_scalar(*col, index.ts_heads); + } else if (key_kind == ExprType::String) { + index.string_heads.partition(partitions); + if (const auto* c_cat = std::get_if>(key)) { + const auto& dict = c_cat->dictionary(); + fill_partitioned_heads( + index.string_heads, n, + [c_cat, &dict](std::size_t r) { + return std::string_view{dict[static_cast(c_cat->code_at(r))]}; + }, + is_null, index.chain_next, index.unique); + } else if (const auto* c_str = std::get_if>(key)) { + fill_partitioned_heads( + index.string_heads, n, + [c_str](std::size_t r) { return std::string_view{(*c_str)[r]}; }, is_null, + index.chain_next, index.unique); + } else { + return std::unexpected("inner join: build-side key type mismatch"); + } + } + return index; +} + +/// The two-key hash build: same chain-of-equal-rows convention as +/// `build_join_hash_index`, over a packed pair of Int64 keys. A row with either +/// key null is never indexed -- null never matches, not even another null. +auto build_join_pair_index(const Column& col0, const Column& col1, + const ValidityBitmap* v0, const ValidityBitmap* v1, + std::size_t partitions) -> JoinHashIndex { + JoinHashIndex index; + index.key_kind = ExprType::Int; + const std::size_t n = col0.size(); + index.chain_next.assign(n, kJoinNil); + const auto* d0 = col0.data(); + const auto* d1 = col1.data(); + index.pair_heads.partition(partitions); + fill_partitioned_heads( + index.pair_heads, n, + [d0, d1](std::size_t r) { + return JoinHashIndex::PairKey{.a = static_cast(d0[r]), + .b = static_cast(d1[r])}; + }, + [v0, v1](std::size_t r) { + return (v0 != nullptr && !(*v0)[r]) || (v1 != nullptr && !(*v1)[r]); + }, + index.chain_next, index.unique); + return index; +} + +/// Which side of a hash join carries the index. +/// +/// The choice is made at RUN time, from measured row counts, and this enum is +/// what makes it a VALUE rather than a shape encoded across three operator +/// members (`mode_`, `probe_op_`, `left_table_`). A build phase +/// can decide it without knowing who will probe, which is what a separately +/// scheduled `HashBuild` needs; see plans/kernel-pipeline-execution-plan.md, +/// "The build-side choice does not block the split" -- the physical plan does +/// not have to name the side statically, because the pipeline that scans the +/// other side is constructed after this phase has already run. +enum class JoinOrientation : std::uint8_t { + BuildRight, ///< index the right side, stream left chunks through it + BuildLeft, ///< index the left side, scan the right once in probe order +}; + +/// Everything a hash build phase decides and produces. Which table to stream +/// and which mode to run in are the caller's derivations from these two, not +/// the build's business. +struct JoinBuildOutcome { + std::shared_ptr index; + JoinOrientation orientation = JoinOrientation::BuildRight; +}; + +/// Build the index over one named side. The primitive both orientation +/// decisions below reduce to, and the only place a `JoinHashIndex` becomes +/// shared and const. +auto build_join_side(const Table& side, const std::string& key_name, ExprType key_kind, + JoinOrientation orientation, std::size_t partitions) + -> std::expected { + auto built = build_join_hash_index(side, key_name, key_kind, partitions); + if (!built.has_value()) { + return std::unexpected(std::move(built.error())); + } + return JoinBuildOutcome{.index = std::make_shared(std::move(*built)), + .orientation = orientation}; +} + +/// The single-key build phase over two already-materialized sides: choose an +/// orientation, build that side's index, return both. Reads and writes no +/// operator state, so the same call serves a join operator and a `HashBuild` +/// that has no probe attached yet. +/// +/// `order_preservation_pays` arrives as a decided bool because it answers a +/// question about the join's OUTPUT plan -- would declining to swap deliver a +/// pending `order` for free -- which is the caller's to answer, not the +/// build's. It is only ever consulted when swapping was otherwise preferred. +auto choose_and_build_single_key(const Table& left, const Table& right, const std::string& left_key, + const std::string& right_key, ExprType key_kind, + bool order_preservation_pays, std::size_t partitions) + -> std::expected { + // Swapping indexes the smaller (left) side and scans the right, which + // gives up left-row order. When an `order` above this join wants exactly + // the order the left already carries, declining to swap delivers it and + // that whole sort disappears -- worth a larger index, but only while + // "larger" stays modest, since the index is probed once per row of the + // other side. The same trade is made in join.cpp. + if (left.rows() < right.rows() && !order_preservation_pays) { + return build_join_side(left, left_key, key_kind, JoinOrientation::BuildLeft, partitions); + } + return build_join_side(right, right_key, key_kind, JoinOrientation::BuildRight, partitions); +} + +/// One side's two Int64 key columns and their validity, or the error a join +/// reports for them. `side_name` is "left" or "right" only so the message +/// keeps naming the side the caller was asking about. +struct PairKeyColumns { + const Column* col0 = nullptr; + const Column* col1 = nullptr; + const ValidityBitmap* v0 = nullptr; + const ValidityBitmap* v1 = nullptr; +}; + +auto pair_key_columns(const Table& side, const std::string& name0, const std::string& name1, + std::string_view side_name) -> std::expected { + const ColumnValue* key0 = side.find(name0); + if (key0 == nullptr) { + return std::unexpected("join key not found in " + std::string(side_name) + + " table: " + name0); + } + const ColumnValue* key1 = side.find(name1); + if (key1 == nullptr) { + return std::unexpected("join key not found in " + std::string(side_name) + + " table: " + name1); + } + PairKeyColumns out; + out.col0 = std::get_if>(key0); + out.col1 = std::get_if>(key1); + if (out.col0 == nullptr || out.col1 == nullptr) { + return std::unexpected( + "ChunkedInnerJoinOperator: two-key join currently requires both keys to be Int64"); + } + const auto* entry0 = side.find_entry(name0); + const auto* entry1 = side.find_entry(name1); + out.v0 = entry0 != nullptr && entry0->validity.has_value() ? &*entry0->validity : nullptr; + out.v1 = entry1 != nullptr && entry1->validity.has_value() ? &*entry1->validity : nullptr; + return out; +} + +/// The two-Int64-key build phase. Same contract as the single-key one, with a +/// simpler decision: no pending-order trade exists on this path, so the +/// smaller side is indexed outright. Validates the right side first, which is +/// the order the operator already checked in, so consolidating the two +/// previously separate blocks cannot change which error a caller sees. +auto choose_and_build_pair(const Table& left, const Table& right, const ir::JoinKey& k0, + const ir::JoinKey& k1, std::size_t partitions) + -> std::expected { + auto right_keys = pair_key_columns(right, k0.right, k1.right, "right"); + if (!right_keys.has_value()) { + return std::unexpected(std::move(right_keys.error())); + } + if (left.rows() <= right.rows()) { + auto left_keys = pair_key_columns(left, k0.left, k1.left, "left"); + if (!left_keys.has_value()) { + return std::unexpected(std::move(left_keys.error())); + } + return JoinBuildOutcome{ + .index = std::make_shared(build_join_pair_index( + *left_keys->col0, *left_keys->col1, left_keys->v0, left_keys->v1, partitions)), + .orientation = JoinOrientation::BuildLeft}; + } + return JoinBuildOutcome{ + .index = std::make_shared(build_join_pair_index( + *right_keys->col0, *right_keys->col1, right_keys->v0, right_keys->v1, partitions)), + .orientation = JoinOrientation::BuildRight}; +} + +/// The probe half of a hash join: everything that consumes a `JoinHashIndex` +/// and turns probe-side rows into join output rows. It owns no build, which is +/// the point -- Phase 4's `HashProbe` has to be able to exist next to a build +/// it did not run, and an operator class that also decides which side to index +/// cannot be that. +/// +/// Held by the operator today rather than scheduled on its own. What the +/// extraction buys now is that the probe's state is enumerable: an index, the +/// join's two output-name plans, the per-worker scratch, and the probe chunk's +/// own validity/dictionary. Nothing else in the join can reach it, and it can +/// reach nothing else in the join. +struct JoinProbe { + const std::vector* keys_ = nullptr; + const ExecutionContext* exec_ = nullptr; + /// The `probe` phase's parallelism, resolved by the caller and set through + /// `ChunkedInnerJoinOperator::bind_probe`. `probe_parallel_workers` reads it + /// rather than re-deriving the floor and the cap. Travels with every copy of + /// this struct (per-worker probes, `JoinProbeOperator`). + physical::BreakerParallelism probe_plan_{}; + ir::JoinSuffixPolicy suffix_; + /// The join's right table, owned jointly and immutably. + /// + /// Shared rather than borrowed because a probe has to be able to outlive + /// the operator that ran the build, and because several probes -- one per + /// morsel worker -- have to be able to read one build side at once. A raw + /// pointer into a member could do neither, and depended on the operator + /// never reallocating the table it points at. Null until the build phase + /// resolves it, which on the deferred path is only after the scan runs. + std::shared_ptr right_; + /// What the build phase produced. Immutable, and shareable: this is the + /// only thing the probe needs from a build. + std::shared_ptr index_; + /// True when the join keys are the two-Int64 pair shape. + bool pair_mode_ = false; + + /// Reset per probe chunk. + const ValidityBitmap* probe_validity_ = nullptr; + /// Probe-side, not build-side: the probe chunk's dictionary code -> build + /// chain head (`kJoinNil` = no match), rebuilt per chunk by + /// `resolve_categorical_heads`. + std::vector probe_code_heads_; + + /// One worker's slice of a parallel probe. Members so the vectors keep + /// their capacity across chunks instead of reallocating per probe. + struct ProbePart { + std::vector li; + std::vector ri; + }; + /// One matching probe row in swapped mode: the right row and the head of + /// the left chain it hit. Phase 2 replays these instead of re-probing. + struct SwappedHit { + std::size_t rrow; + std::size_t head; ///< first left row in the chain for this key + }; + /// One worker's slice of a swapped-mode phase 1. A member for the same + /// reason as `ProbePart`: capacity survives across chunks. + struct SwappedPart { + std::vector hits; + std::size_t total = 0; ///< output rows this part's chains expand to + }; + std::vector probe_parts_; + std::vector swapped_parts_; + std::vector part_offsets_; + std::vector right_emit_idx_; + std::vector right_emit_names_; + std::vector left_emit_names_; + std::optional columns_; + bool right_emit_ready_ = false; + + static constexpr std::size_t kNil = kJoinNil; + + /// The build this probe reads. Never null once a build side has been + /// chosen; the `Precomputed` mode returns before one exists. + [[nodiscard]] auto index() const noexcept -> const JoinHashIndex& { return *index_; } + + [[nodiscard]] auto probe_is_null(std::size_t row) const noexcept -> bool { + return probe_validity_ != nullptr && !(*probe_validity_)[row]; + } + + // Which right columns this join emits, and under which names. Both come + // from the shared planner (ir/join_output.hpp), so the chunked route lands + // on the same output schema as the materialized route and IR inference. + // The left column names are identical for every chunk, so the plan is + // computed once from the first assembled chunk. + + auto setup_right_emit_schema(const Table& left_side) -> std::expected { + if (right_emit_ready_) { + return {}; + } + const auto left_names = table_column_names(left_side); + const auto right_names = table_column_names(*right_); + auto concrete = + ir::resolve_join_columns(ir::JoinKind::Inner, *keys_, left_names, right_names, suffix_); + if (!concrete.has_value()) { + return std::unexpected(std::move(concrete.error())); + } + const bool concrete_layout_matches_plan = + columns_.has_value() && columns_->left_input_names.size() == left_names.size() && + columns_->right_input_names.size() == right_names.size() && + std::ranges::equal(columns_->left_input_names, left_names) && + std::ranges::equal(columns_->right_input_names, right_names); + if (!concrete_layout_matches_plan) { + // A pushed-down predicate may consume a column inside a lazy child + // and omit it from the join input. Rebind the complete key/output + // mapping at that concrete boundary once; probe and gather loops + // remain positional. + columns_ = std::move(*concrete); + } else if (*columns_ != *concrete) { + return std::unexpected( + "physical join column mapping does not match its concrete inputs"); + } + if (columns_->keys.size() != keys_->size()) { + return std::unexpected("physical join column mapping has the wrong key count"); + } + for (std::size_t i = 0; i < keys_->size(); ++i) { + const ir::JoinKeyColumns& mapped = columns_->keys[i]; + if (mapped.left_index >= left_side.columns.size() || + mapped.right_index >= right_->columns.size() || + left_side.columns[mapped.left_index].name != keys_->at(i).left || + right_->columns[mapped.right_index].name != keys_->at(i).right) { + return std::unexpected( + "physical join column mapping does not match its concrete inputs"); + } + } + const std::vector& plan = columns_->output; + // A suffix clause renames the *left* side of a collision too, so the + // left names come from the plan as well; taking them from the chunk + // would keep the pre-rename spelling. + left_emit_names_.reserve(left_side.columns.size()); + for (const auto& column : plan) { + if (column.side == ir::JoinOutputSide::Left) { + left_emit_names_.push_back(column.name); + } + } + right_emit_idx_.reserve(plan.size() - left_side.columns.size()); + right_emit_names_.reserve(plan.size() - left_side.columns.size()); + for (const auto& column : plan) { + if (column.side != ir::JoinOutputSide::Right) { + continue; + } + right_emit_idx_.push_back(column.source_index); + right_emit_names_.push_back(column.name); + } + right_emit_ready_ = true; + return {}; + } + + /// Run `body(begin, end, li, ri)` over the probe rows across workers, + /// concatenating each range's output in range order. Returns false when the + /// parallel path declines and the caller should run its serial loop. + /// + /// **This is the join's only parallel axis, and it is the whole of it.** + /// The build side is a shared read-only hash index — `heads`, `index().chain_next` + /// — so probing it concurrently needs no locking at all, and the build + /// itself is not worth threading: it is 1.5% of q10 against the probe and + /// output assembly's ~15%. + /// + /// **Per-worker output rather than count-then-fill.** The obvious shape is + /// to count matches per range, prefix-sum, then have each worker write its + /// slice — but that probes the hash table TWICE per row, and a redundant + /// cache-missing lookup per probe row is exactly the cost `emit_swapped` + /// was restructured to avoid (q03 probes 3.2M lineitems to emit ~30K rows). + /// Each worker appends to its own vectors instead and they are concatenated + /// afterwards: one memcpy of two size_t arrays, against one hash probe per + /// row saved. + /// + /// Order is exactly the serial order — ranges are contiguous and visited in + /// order, and each range appends in row order — so the output is + /// byte-identical however the workers interleave. + /// The shared admission gate for every parallel probe axis: how many + /// workers a probe over `n` rows may fan out to, or 0 to decline and run + /// the caller's serial loop. + /// + /// The floor (`1U << 14U`) and the worker cap (`min(compute_budget, pool, + /// 64)`) are the `probe` phase of the physical plan + /// (`physical::join_probe_parallelism`), read from `probe_plan_`. + /// src/runtime/PARALLELISM.md, "Target: parallelism as a plan decision". + /// The checks that stay here are the ones only the operator can make: + /// `parallel_join_probe` (a feature toggle, kept operator-side like the hash + /// build's `IBEX_JOIN_BUILD_SERIAL`), nesting (`on_worker_pool_thread` -- no + /// nested pool submissions), and whether *this chunk* cleared the floor (a + /// streamed probe side's per-chunk row count is not a plan-time fact). + [[nodiscard]] auto probe_parallel_workers(std::size_t n) const -> std::size_t { + if (exec_ == nullptr || !exec_->parallel_join_probe || on_worker_pool_thread() || + probe_plan_.decline != physical::FanOutDecline::None || probe_plan_.worker_cap < 2 || + n < probe_plan_.row_floor) { + return 0; + } + return probe_plan_.worker_cap; + } + + template + auto probe_ranges_parallel(std::size_t n, std::vector& li, + std::vector& ri, const Body& body) -> bool { + const std::size_t workers = probe_parallel_workers(n); + if (workers == 0) { + return false; + } + auto& pool = process_worker_pool(); + const std::size_t grain = (n + workers - 1) / workers; + probe_parts_.resize(workers); + { + auto batch = pool.submit(workers, [&](std::size_t w) { + auto& part = probe_parts_[w]; + part.li.clear(); + part.ri.clear(); + const std::size_t begin = w * grain; + const std::size_t end = std::min(n, begin + grain); + if (begin >= end) { + return; + } + // Reserve for the common case of roughly one match per row; + // a fan-out join grows past it, which is what a vector is for. + part.li.reserve(end - begin); + part.ri.reserve(end - begin); + body(begin, end, part.li, part.ri); + }); + batch.wait(); + } + std::size_t total = 0; + for (const auto& part : probe_parts_) { + total += part.li.size(); + } + li.resize(total); + ri.resize(total); + // The concat is the price the fan-out pays that the serial probe does + // not, and on a high-match join it is the whole regression: every row + // matching means `total == n`, i.e. two full index arrays copied + // again. Each part's destination slice is disjoint and known, so the + // copies go back to the workers; below the threshold the batch costs + // more than the memcpy it spreads. + constexpr std::size_t kMinParallelConcatRows = 1U << 16U; + const auto copy_part = [&](std::size_t w, std::size_t at) { + const auto& part = probe_parts_[w]; + std::ranges::copy(part.li, li.begin() + static_cast(at)); + std::ranges::copy(part.ri, ri.begin() + static_cast(at)); + }; + part_offsets_.resize(workers); + std::size_t at = 0; + for (std::size_t w = 0; w < workers; ++w) { + part_offsets_[w] = at; + at += probe_parts_[w].li.size(); + } + if (total >= kMinParallelConcatRows) { + auto batch = + pool.submit(workers, [&](std::size_t w) { copy_part(w, part_offsets_[w]); }); + batch.wait(); + } else { + for (std::size_t w = 0; w < workers; ++w) { + copy_part(w, part_offsets_[w]); + } + } + if (exec_->parallel_stats != nullptr) { + exec_->parallel_stats->parallel_probes.fetch_add(1, std::memory_order_relaxed); + } + return true; + } + + /// Swapped-mode probe: phase 1 walks right rows `head_of` resolves against + /// the left index, phase 2 expands the recorded chains into (li, ri). + /// The parallel path fans phase 1 out over contiguous right-row ranges and + /// phase 2 out over the per-range hit lists — each part's output slice + /// starts at the prefix sum of the parts before it, so workers write + /// disjoint slices and the result is byte-identical to the serial replay + /// (parts are visited in range order, ranges in row order). + template + void probe_swapped(std::size_t n_right, const HeadOf& head_of, std::vector& li, + std::vector& ri) { + const auto scan = [&](std::size_t begin, std::size_t end, std::vector& hits, + std::size_t& total) { + for (std::size_t r = begin; r < end; ++r) { + if (probe_is_null(r)) { + continue; + } + const std::size_t head = head_of(r); + if (head == kNil) { + continue; + } + hits.push_back(SwappedHit{.rrow = r, .head = head}); + for (std::size_t cur = head; cur != kNil; cur = index().chain_next[cur]) { + ++total; + } + } + }; + const auto replay = [&](const std::vector& hits, std::size_t pos) { + for (const SwappedHit& hit : hits) { + for (std::size_t cur = hit.head; cur != kNil; cur = index().chain_next[cur]) { + li[pos] = cur; + ri[pos] = hit.rrow; + ++pos; + } + } + }; + + const std::size_t workers = probe_parallel_workers(n_right); + if (workers == 0) { + std::vector hits; + std::size_t total = 0; + scan(0, n_right, hits, total); + li.assign(total, 0); + ri.assign(total, 0); + replay(hits, 0); + return; + } + auto& pool = process_worker_pool(); + const std::size_t grain = (n_right + workers - 1) / workers; + swapped_parts_.resize(workers); + { + auto batch = pool.submit(workers, [&](std::size_t w) { + auto& part = swapped_parts_[w]; + part.hits.clear(); + part.total = 0; + const std::size_t begin = w * grain; + const std::size_t end = std::min(n_right, begin + grain); + if (begin < end) { + scan(begin, end, part.hits, part.total); + } + }); + batch.wait(); + } + part_offsets_.resize(workers); + std::size_t total = 0; + for (std::size_t w = 0; w < workers; ++w) { + part_offsets_[w] = total; + total += swapped_parts_[w].total; + } + li.assign(total, 0); + ri.assign(total, 0); + { + auto batch = pool.submit( + workers, [&](std::size_t w) { replay(swapped_parts_[w].hits, part_offsets_[w]); }); + batch.wait(); + } + if (exec_->parallel_stats != nullptr) { + exec_->parallel_stats->parallel_probes.fetch_add(1, std::memory_order_relaxed); + } + } + + // Stream mode: walk the probe side (a left chunk), for each row look + // up the right-keyed chain and append (li, ri) in probe-scan order. + // Returns true if every probe row matched exactly once (li == 0..n-1). + // Only possible when the build side was unique; otherwise falls back + // to the chained walk. + template + auto probe_scalar(const Map& heads, std::size_t n, GetKey get, std::vector& li, + std::vector& ri) -> bool { + // One body for both paths, so the parallel and serial results cannot + // drift: the parallel one runs it per range, the serial one once. + const auto scan = [&](std::size_t begin, std::size_t end, std::vector& out_l, + std::vector& out_r) { + for (std::size_t l = begin; l < end; ++l) { + if (probe_is_null(l)) { + continue; + } + const std::size_t head = heads.find_head(get(l)); + if (head == kNil) { + continue; + } + for (std::size_t cur = head; cur != kNil; cur = index().chain_next[cur]) { + out_l.push_back(l); + out_r.push_back(cur); + } + } + }; + if (probe_ranges_parallel(n, li, ri, scan)) { + // `li_identity` means li == 0..n-1, which for a unique build side + // is exactly "every row matched" — the same test the serial path + // makes, just recovered from the totals. + return index().unique && li.size() == n; + } + if (index().unique) { + li.resize(n); + ri.resize(n); + std::size_t* lp = li.data(); + std::size_t* rp = ri.data(); + std::size_t out = 0; + for (std::size_t l = 0; l < n; ++l) { + if (probe_is_null(l)) { + continue; + } + const std::size_t head = heads.find_head(get(l)); + if (head == kNil) { + continue; + } + lp[out] = l; + rp[out] = head; + ++out; + } + li.resize(out); + ri.resize(out); + return out == n; + } + for (std::size_t l = 0; l < n; ++l) { + if (probe_is_null(l)) { + continue; + } + const std::size_t head = heads.find_head(get(l)); + if (head == kNil) { + continue; + } + std::size_t cur = head; + while (cur != kNil) { + li.push_back(l); + ri.push_back(cur); + cur = index().chain_next[cur]; + } + } + return false; + } + + // Probe with the build-side chain head already resolved per row. Same two + // shapes as `probe_scalar`, but the caller supplies the head instead of a + // key to hash — see `resolve_categorical_heads`. + template + auto probe_resolved(std::size_t n, GetHead head_of, std::vector& li, + std::vector& ri) -> bool { + const auto scan = [&](std::size_t begin, std::size_t end, std::vector& out_l, + std::vector& out_r) { + for (std::size_t l = begin; l < end; ++l) { + if (probe_is_null(l)) { + continue; + } + for (std::size_t cur = head_of(l); cur != kNil; cur = index().chain_next[cur]) { + out_l.push_back(l); + out_r.push_back(cur); + } + } + }; + if (probe_ranges_parallel(n, li, ri, scan)) { + return index().unique && li.size() == n; + } + if (index().unique) { + li.resize(n); + ri.resize(n); + std::size_t* lp = li.data(); + std::size_t* rp = ri.data(); + std::size_t out = 0; + for (std::size_t l = 0; l < n; ++l) { + if (probe_is_null(l)) { + continue; + } + const std::size_t head = head_of(l); + if (head == kNil) { + continue; + } + lp[out] = l; + rp[out] = head; + ++out; + } + li.resize(out); + ri.resize(out); + return out == n; + } + for (std::size_t l = 0; l < n; ++l) { + if (probe_is_null(l)) { + continue; + } + std::size_t cur = head_of(l); + while (cur != kNil) { + li.push_back(l); + ri.push_back(cur); + cur = index().chain_next[cur]; + } + } + return false; + } + + // A Categorical probe column is a dictionary plus one code per row, so + // every row sharing a code resolves to the same build chain. Resolve the + // DICTIONARY against the build index once — 252 entries for the symbol + // join — instead of rebuilding a string_view and hashing plus memcmp'ing + // it per row across 8M rows. That lookup was the single largest cost in + // the join profile (robin_hood string probe + __memcmp_avx2 + _Hash_bytes + // together ~57%). + // + // Rebuilt per chunk rather than cached on the operator: chunks of one scan + // usually share a dictionary, but nothing in the type guarantees it, and a + // stale memo would silently join against the wrong rows. |dict| lookups + // per chunk is noise next to |chunk rows|. + void resolve_categorical_heads(const std::vector& dict) { + probe_code_heads_.assign(dict.size(), kNil); + for (std::size_t c = 0; c < dict.size(); ++c) { + probe_code_heads_[c] = index().string_heads.find_head(std::string_view{dict[c]}); + } + } + + auto probe_chunk_against_right(Table left_chunk) -> std::expected { + if (auto mapped = setup_right_emit_schema(left_chunk); !mapped.has_value()) { + return std::unexpected(std::move(mapped.error())); + } + if (!columns_.has_value()) { + return std::unexpected("join column mapping not bound"); + } + if (pair_mode_) { + return probe_chunk_pair(std::move(left_chunk)); + } + const ir::JoinKeyColumns& key_columns = columns_->keys.front(); + const ColumnEntry& probe_entry = left_chunk.columns[key_columns.left_index]; + const ColumnValue* key = probe_entry.column.get(); + probe_validity_ = probe_entry.validity.has_value() ? &*probe_entry.validity : nullptr; + + std::vector li; + std::vector ri; + const std::size_t n = left_chunk.rows(); + li.reserve(n); + ri.reserve(n); + bool li_identity = false; + + if (index().key_kind == ExprType::Int) { + const auto* col = std::get_if>(key); + if (col == nullptr) { + return std::unexpected("inner join: left key type mismatch"); + } + const auto* data = col->data(); + li_identity = + probe_scalar(index().i64_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); + } else if (index().key_kind == ExprType::Double) { + const auto* col = std::get_if>(key); + if (col == nullptr) { + return std::unexpected("inner join: left key type mismatch"); + } + const auto* data = col->data(); + li_identity = + probe_scalar(index().f64_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); + } else if (index().key_kind == ExprType::Bool) { + const auto* col = std::get_if>(key); + if (col == nullptr) { + return std::unexpected("inner join: left key type mismatch"); + } + li_identity = probe_scalar( + index().bool_heads, n, [&](std::size_t i) { return (*col)[i]; }, li, ri); + } else if (index().key_kind == ExprType::Date) { + const auto* col = std::get_if>(key); + if (col == nullptr) { + return std::unexpected("inner join: left key type mismatch"); + } + const auto* data = col->data(); + li_identity = + probe_scalar(index().date_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); + } else if (index().key_kind == ExprType::Timestamp) { + const auto* col = std::get_if>(key); + if (col == nullptr) { + return std::unexpected("inner join: left key type mismatch"); + } + const auto* data = col->data(); + li_identity = + probe_scalar(index().ts_heads, n, [&](std::size_t i) { return data[i]; }, li, ri); + } else if (index().key_kind == ExprType::String) { + if (const auto* c_cat = std::get_if>(key)) { + const auto& dict = c_cat->dictionary(); + // Resolving the dictionary costs |dict| hash lookups and saves + // one per row, so it pays exactly when the dictionary is + // smaller than the chunk. A dictionary larger than the chunk + // (a narrow slice of a high-cardinality column) would hash more + // keys than there are rows to answer. + if (dict.size() < n) { + resolve_categorical_heads(dict); + li_identity = probe_resolved( + n, + [&](std::size_t i) { + return probe_code_heads_[static_cast(c_cat->code_at(i))]; + }, + li, ri); + } else { + li_identity = probe_scalar( + index().string_heads, n, + [&](std::size_t i) { + return std::string_view{ + dict[static_cast(c_cat->code_at(i))]}; + }, + li, ri); + } + } else if (const auto* c_str = std::get_if>(key)) { + li_identity = probe_scalar( + index().string_heads, n, [&](std::size_t i) { return (*c_str)[i]; }, li, ri); + } else { + return std::unexpected("inner join: left key type mismatch"); + } + } + + const std::size_t total = li_identity ? ri.size() : li.size(); + return assemble_output(std::move(left_chunk), li.data(), ri.data(), total, li_identity); + } + + auto probe_chunk_pair(Table left_chunk) -> std::expected { + if (!columns_.has_value()) { + return std::unexpected("join column mapping not bound"); + } + const ir::JoinKeyColumns& k0 = columns_->keys[0]; + const ir::JoinKeyColumns& k1 = columns_->keys[1]; + const ColumnEntry& e0 = left_chunk.columns[k0.left_index]; + const ColumnEntry& e1 = left_chunk.columns[k1.left_index]; + const ColumnValue* key0 = e0.column.get(); + const ColumnValue* key1 = e1.column.get(); + const auto* col0 = std::get_if>(key0); + const auto* col1 = std::get_if>(key1); + if (col0 == nullptr || col1 == nullptr) { + return std::unexpected( + "inner join: left key type mismatch (two-key join expects " + "Int64)"); + } + const ValidityBitmap* v0 = e0.validity.has_value() ? &*e0.validity : nullptr; + const ValidityBitmap* v1 = e1.validity.has_value() ? &*e1.validity : nullptr; + + std::vector li; + std::vector ri; + const std::size_t n = left_chunk.rows(); + li.reserve(n); + ri.reserve(n); + + const auto* d0 = col0->data(); + const auto* d1 = col1->data(); + const auto is_null = [&](std::size_t r) { + return (v0 != nullptr && !(*v0)[r]) || (v1 != nullptr && !(*v1)[r]); + }; + const auto get_key = [&](std::size_t r) { + return JoinHashIndex::PairKey{.a = static_cast(d0[r]), + .b = static_cast(d1[r])}; + }; + const bool li_identity = probe_pair(n, is_null, get_key, li, ri); + + const std::size_t total = li_identity ? ri.size() : li.size(); + return assemble_output(std::move(left_chunk), li.data(), ri.data(), total, li_identity); + } + + // Same two shapes as `probe_scalar` (parallel fan-out via + // `probe_ranges_parallel`, then a unique-build fast path, then the + // general chained walk) but with an explicit null check instead of the + // single-bitmap `probe_is_null` member, since a probe row here is null + // when EITHER key is. + template + auto probe_pair(std::size_t n, IsNull is_null, GetKey get_key, std::vector& li, + std::vector& ri) -> bool { + const auto scan = [&](std::size_t begin, std::size_t end, std::vector& out_l, + std::vector& out_r) { + for (std::size_t l = begin; l < end; ++l) { + if (is_null(l)) { + continue; + } + const std::size_t head = index().pair_heads.find_head(get_key(l)); + if (head == kNil) { + continue; + } + for (std::size_t cur = head; cur != kNil; cur = index().chain_next[cur]) { + out_l.push_back(l); + out_r.push_back(cur); + } + } + }; + if (probe_ranges_parallel(n, li, ri, scan)) { + return index().unique && li.size() == n; + } + if (index().unique) { + li.resize(n); + ri.resize(n); + std::size_t* lp = li.data(); + std::size_t* rp = ri.data(); + std::size_t out = 0; + for (std::size_t l = 0; l < n; ++l) { + if (is_null(l)) { + continue; + } + const std::size_t head = index().pair_heads.find_head(get_key(l)); + if (head == kNil) { + continue; + } + lp[out] = l; + rp[out] = head; + ++out; + } + li.resize(out); + ri.resize(out); + return out == n; + } + for (std::size_t l = 0; l < n; ++l) { + if (is_null(l)) { + continue; + } + const std::size_t head = index().pair_heads.find_head(get_key(l)); + if (head == kNil) { + continue; + } + std::size_t cur = head; + while (cur != kNil) { + li.push_back(l); + ri.push_back(cur); + cur = index().chain_next[cur]; + } + } + return false; + } + + // Swapped mode: the hash index is on the left table, so the right table + // is the probe side, and output must still come out in left-row order. + // + // Phase 1 probes each right row once and remembers the head of every left + // chain it hit; phase 2 replays just those hits to fill (li, ri). The hash + // table is therefore probed once per right row for the whole join, not + // once per phase: a selective join over a large right side (q03 probes + // 3.2M lineitems to emit ~30K rows) no longer pays for 3.2M redundant + // cache-missing lookups. `hits` costs one entry per *matching* right row, + // so it is bounded by the output row count. + auto emit_swapped(const Table& left_table) -> std::expected { + if (auto mapped = setup_right_emit_schema(left_table); !mapped.has_value()) { + return std::unexpected(std::move(mapped.error())); + } + if (!columns_.has_value()) { + return std::unexpected("join column mapping not bound"); + } + if (pair_mode_) { + return emit_swapped_pair(left_table); + } + const ir::JoinKeyColumns& key_columns = columns_->keys.front(); + const ColumnEntry& right_entry = right_->columns[key_columns.right_index]; + const ColumnValue* rkey = right_entry.column.get(); + const std::size_t n_right = right_->rows(); + + // In swapped mode the index is on the left, so the right table is the + // probe side. Its null-keyed rows match nothing (see build_join_hash_index). + probe_validity_ = right_entry.validity.has_value() ? &*right_entry.validity : nullptr; + + std::vector li; + std::vector ri; + + // Every key kind reduces to "resolve right row r to a left chain head + // or kNil"; the map branches wrap the hash lookup, the categorical + // fast path hands the pre-resolved head straight through. One shape + // means `probe_swapped` is the single scan/replay implementation for + // both the serial and the parallel path. + auto do_phase1 = [&](auto&& key_at, const auto& heads) { + probe_swapped( + n_right, [&](std::size_t r) { return heads.find_head(key_at(r)); }, li, ri); + }; + // Same shape with the chain head already resolved — see + // `resolve_categorical_heads`. + auto do_phase1_resolved = [&](auto&& head_at) { probe_swapped(n_right, head_at, li, ri); }; + + if (index().key_kind == ExprType::Int) { + const auto* col = std::get_if>(rkey); + if (col == nullptr) + return std::unexpected("inner join: right key type mismatch"); + const auto* data = col->data(); + do_phase1([&](std::size_t r) { return data[r]; }, index().i64_heads); + } else if (index().key_kind == ExprType::Double) { + const auto* col = std::get_if>(rkey); + if (col == nullptr) + return std::unexpected("inner join: right key type mismatch"); + const auto* data = col->data(); + do_phase1([&](std::size_t r) { return data[r]; }, index().f64_heads); + } else if (index().key_kind == ExprType::Bool) { + const auto* col = std::get_if>(rkey); + if (col == nullptr) + return std::unexpected("inner join: right key type mismatch"); + do_phase1([&](std::size_t r) { return (*col)[r]; }, index().bool_heads); + } else if (index().key_kind == ExprType::Date) { + const auto* col = std::get_if>(rkey); + if (col == nullptr) + return std::unexpected("inner join: right key type mismatch"); + const auto* data = col->data(); + do_phase1([&](std::size_t r) { return data[r]; }, index().date_heads); + } else if (index().key_kind == ExprType::Timestamp) { + const auto* col = std::get_if>(rkey); + if (col == nullptr) + return std::unexpected("inner join: right key type mismatch"); + const auto* data = col->data(); + do_phase1([&](std::size_t r) { return data[r]; }, index().ts_heads); + } else if (index().key_kind == ExprType::String) { + if (const auto* c_cat = std::get_if>(rkey)) { + const auto& dict = c_cat->dictionary(); + if (dict.size() < n_right) { + resolve_categorical_heads(dict); + do_phase1_resolved([&](std::size_t r) { + return probe_code_heads_[static_cast(c_cat->code_at(r))]; + }); + } else { + do_phase1( + [&](std::size_t r) { + return std::string_view{ + dict[static_cast(c_cat->code_at(r))]}; + }, + index().string_heads); + } + } else if (const auto* c_str = std::get_if>(rkey)) { + do_phase1([&](std::size_t r) { return (*c_str)[r]; }, index().string_heads); + } else { + return std::unexpected("inner join: right key type mismatch"); + } + } + + // Output order is the order phase 1 visited the hits — right-scan + // (probe) order. Row order is outside the join contract (SPEC.md + // §5.6), so there's no correctness reason to reassemble by left row + // instead; doing so was actively harmful, permuting the output away + // from the probe side's natural scan order and hurting cache locality + // on any downstream join that probes this join's output. + Table left_copy; + left_copy.columns.reserve(left_table.columns.size()); + for (const auto& c : left_table.columns) { + left_copy.add_column(c.name, *c.column); + left_copy.columns.back().validity = c.validity; + } + return assemble_output(std::move(left_copy), li.data(), ri.data(), li.size()); + } + + // Swapped pair-mode counterpart of `emit_swapped`: the pair index is on + // the build-side left table, so the right table's rows are the probe + // side. Reuses + // `probe_swapped` unchanged -- it is already generic over a + // `head_of(row)` callback -- with the null check folded into `head_of` + // itself (returning `kNil`) instead of the single-bitmap `probe_is_null` + // member, since a row here is null when EITHER key is. + auto emit_swapped_pair(const Table& left_table) -> std::expected { + if (!columns_.has_value()) { + return std::unexpected("join column mapping not bound"); + } + const ir::JoinKeyColumns& k0 = columns_->keys[0]; + const ir::JoinKeyColumns& k1 = columns_->keys[1]; + const ColumnEntry& e0 = right_->columns[k0.right_index]; + const ColumnEntry& e1 = right_->columns[k1.right_index]; + const ColumnValue* rkey0 = e0.column.get(); + const ColumnValue* rkey1 = e1.column.get(); + const auto* col0 = std::get_if>(rkey0); + const auto* col1 = std::get_if>(rkey1); + if (col0 == nullptr || col1 == nullptr) { + return std::unexpected( + "inner join: right key type mismatch (two-key join expects Int64)"); + } + const ValidityBitmap* v0 = e0.validity.has_value() ? &*e0.validity : nullptr; + const ValidityBitmap* v1 = e1.validity.has_value() ? &*e1.validity : nullptr; + const auto* d0 = col0->data(); + const auto* d1 = col1->data(); + const std::size_t n_right = right_->rows(); + + const auto head_of = [&](std::size_t r) -> std::size_t { + if ((v0 != nullptr && !(*v0)[r]) || (v1 != nullptr && !(*v1)[r])) { + return kNil; + } + const JoinHashIndex::PairKey key{.a = static_cast(d0[r]), + .b = static_cast(d1[r])}; + return index().pair_heads.find_head(key); + }; + + std::vector li; + std::vector ri; + probe_swapped(n_right, head_of, li, ri); + + Table left_copy; + left_copy.columns.reserve(left_table.columns.size()); + for (const auto& c : left_table.columns) { + left_copy.add_column(c.name, *c.column); + left_copy.columns.back().validity = c.validity; + } + return assemble_output(std::move(left_copy), li.data(), ri.data(), li.size()); + } + + auto assemble_output(Table left_side, const std::size_t* li, const std::size_t* ri, + std::size_t total, bool li_identity = false, bool ri_identity = false) + -> std::expected { + Table output; + if (!right_emit_ready_) { + if (auto ready = setup_right_emit_schema(left_side); !ready.has_value()) { + return std::unexpected(std::move(ready.error())); + } + } + output.columns.reserve(left_side.columns.size() + right_emit_idx_.size()); + + // A stream with no matches still has a schema. Returning a bare empty + // table here used to be harmless only because the regular chunked path + // normally has another node to provide one; the whole-table adapter + // must be equivalent to join_table_impl even for an empty result. + if (total == 0) { + for (std::size_t i = 0; i < left_side.columns.size(); ++i) { + output.add_column(std::string(left_emit_names_[i]), + make_empty_like(*left_side.columns[i].column)); + } + for (std::size_t e = 0; e < right_emit_idx_.size(); ++e) { + output.add_column(std::string(right_emit_names_[e]), + make_empty_like(*right_->columns[right_emit_idx_[e]].column)); + } + return output; + } + + // Gather a batch of columns in ONE fan-out. Calling `gather_column` per + // column instead lets each call fan out its own rows, which submits and + // waits a batch PER COLUMN — see `gather_columns_batched` for the + // measurement that ruled that out. This is an inner join, so no index + // carries a `kNull` sentinel and no job needs `indivisible`. + const auto gather_batch = + [&](std::span jobs) -> std::vector { + return gather_columns_batched(jobs, total, exec_, [&](std::size_t j) -> GatheredColumn { + const auto& job = jobs[j]; + ColumnValue gathered = gather_column(*job.column, job.idx, total, nullptr); + std::optional val; + if (job.validity != nullptr) { + ValidityBitmap dst(total, false); + gather_validity_range(dst, *job.validity, + std::span{job.idx, total}, 0, total); + val = std::move(dst); + } + return {std::move(gathered), std::move(val)}; + }); + }; + + // li_identity: every probe row matched exactly once, so left columns + // can be passed through directly (shared_ptr share) instead of + // gathered. Do NOT move the underlying ColumnValue — the shared_ptr + // may be aliased by upstream state (e.g., re-runnable source). + const auto left_name = [&](std::size_t i, const ColumnEntry& lc) -> const std::string& { + return i < left_emit_names_.size() ? left_emit_names_[i] : lc.name; + }; + + // The left's ordering, restated in the output's names, when this batch + // emitted the left rows in their own order. Same rule and reasoning as + // the materialized join in join.cpp -- a join promises no order, but a + // path that produces one should say so, and the claim is proved from + // the emitted index array rather than from which mode ran. Computed + // before the identity branch below, which renames left columns in place. + const auto carried_ordering = [&]() -> std::vector { + if (!left_side.properties().ordering().has_value()) { + return {}; + } + if (!li_identity) { + for (std::size_t i = 1; i < total; ++i) { + if (li[i] < li[i - 1]) { + return {}; + } + } + } + std::vector out; + for (const auto& key : *left_side.properties().ordering()) { + std::optional emitted; + for (std::size_t i = 0; i < left_side.columns.size(); ++i) { + if (left_side.columns[i].name == key.name) { + emitted = left_name(i, left_side.columns[i]); + break; + } + } + if (!emitted.has_value()) { + return {}; // a key the output cannot name + } + out.push_back( + ir::OrderKey{.name = std::move(*emitted), .ascending = key.ascending}); + } + return out; + }(); + if (li_identity && total == left_side.rows()) { + for (std::size_t i = 0; i < left_side.columns.size(); ++i) { + auto& lc = left_side.columns[i]; + std::string name = left_name(i, lc); + lc.name = name; + output.index[std::move(name)] = output.columns.size(); + output.columns.push_back(std::move(lc)); + } + } else { + std::vector jobs; + jobs.reserve(left_side.columns.size()); + for (const auto& lc : left_side.columns) { + jobs.push_back({.column = lc.column.get(), + .validity = lc.validity.has_value() ? &*lc.validity : nullptr, + .idx = li, + .indivisible = false}); + } + auto gathered = gather_batch(jobs); + for (std::size_t i = 0; i < left_side.columns.size(); ++i) { + const auto& lc = left_side.columns[i]; + if (gathered[i].second.has_value()) { + output.add_column(left_name(i, lc), std::move(gathered[i].first), + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + std::move(*gathered[i].second)); + } else { + output.add_column(left_name(i, lc), std::move(gathered[i].first)); + } + } + } + + // ri_identity: every emitted row consumes the next probe-side row + // exactly once (two-phase deferred probe with a unique build side), + // so probe columns are shared rather than gathered — the same + // reasoning as li_identity above. + const bool share_right = ri_identity && total == right_->rows(); + if (share_right) { + for (std::size_t e = 0; e < right_emit_idx_.size(); ++e) { + output.add_column_from(std::string(right_emit_names_[e]), + right_->columns[right_emit_idx_[e]]); + } + } else { + std::vector jobs; + jobs.reserve(right_emit_idx_.size()); + for (const auto index : right_emit_idx_) { + const auto& rc = right_->columns[index]; + jobs.push_back({.column = rc.column.get(), + .validity = rc.validity.has_value() ? &*rc.validity : nullptr, + .idx = ri, + .indivisible = false}); + } + auto gathered = gather_batch(jobs); + for (std::size_t e = 0; e < right_emit_idx_.size(); ++e) { + std::string name = right_emit_names_[e]; + if (gathered[e].second.has_value()) { + output.add_column(std::move(name), std::move(gathered[e].first), + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + std::move(*gathered[e].second)); + } else { + output.add_column(std::move(name), std::move(gathered[e].first)); + } + } + } + if (!carried_ordering.empty()) { + output.set_properties(output.properties().with_ordering(carried_ordering)); + } + return output; + } +}; + +/// The streaming probe as an operator: a source of probe-side chunks, one +/// completed build, and nothing else. +/// +/// This is Phase 4's `HashProbe`. It owns no build -- it reads one through +/// `JoinProbe`'s `shared_ptr` handles -- and it does not know which +/// side of the join was hashed, because by the time it exists that is settled. +/// Two things follow, and they are the reason it is a type rather than a loop +/// inside the join: it can be constructed next to a build it did not run, and +/// several of it can read one build at once, which is what a per-worker morsel +/// chain needs. +/// +/// The empty-schema carrier travels with it, because "this join produced no +/// rows but still has a schema" is a property of probing, not of the operator +/// that decided the orientation. +class JoinProbeOperator final : public Operator { + public: + /// `preserve_empty_morsels` is the same contract every map kernel in a + /// morsel chain honours: one input morsel yields exactly one identified + /// output morsel, because the ordered ring indexes by sequence and a + /// coalesced empty result would be a lost slot rather than a smaller + /// answer. A probe needs it more than a filter does -- a morsel of + /// probe-side rows that matches nothing is entirely ordinary. + JoinProbeOperator(OperatorPtr child, JoinProbe probe, bool preserve_empty_morsels = false) + : child_(std::move(child)), + probe_(std::move(probe)), + preserve_empty_(preserve_empty_morsels) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + while (true) { + auto chunk_res = child_->next(); + if (!chunk_res.has_value()) { + return std::unexpected(std::move(chunk_res.error())); + } + if (!chunk_res.value().has_value()) { + if (!emitted_nonempty_ && empty_schema_.has_value()) { + auto schema = std::move(*empty_schema_); + empty_schema_.reset(); + return std::optional{table_to_chunk(std::move(schema))}; + } + return std::optional{}; + } + Chunk input = std::move(*chunk_res.value()); + // A morsel's identity travels with it. `sequence` and `row_offset` + // identify which morsel this is, not which rows it holds, so a + // probe propagates them unchanged exactly as a filter does -- + // both change the row count, and neither changes which morsel it + // is answering for. The ordered ring rejects a chunk that arrives + // without them. + const std::uint64_t sequence = input.sequence; + const std::size_t row_offset = input.row_offset; + auto out = probe_.probe_chunk_against_right(chunk_to_table(std::move(input))); + if (!out.has_value()) { + return std::unexpected(std::move(out.error())); + } + if (out->rows() == 0 && !preserve_empty_) { + // Keep the planned empty table as a schema carrier. A join + // with no matches still has its left and right output columns; + // without this, a materializing sink sees no chunks at all. + empty_schema_ = std::move(*out); + continue; + } + emitted_nonempty_ = true; + Chunk result = table_to_chunk(std::move(*out)); + result.sequence = sequence; + result.row_offset = row_offset; + return std::optional{std::move(result)}; + } + } + + private: + OperatorPtr child_; + JoinProbe probe_; + bool preserve_empty_ = false; + std::optional
empty_schema_; + bool emitted_nonempty_ = false; +}; + +auto make_probe_factory(JoinProbe probe) -> JoinProbeFactory { + const ExecutionContext* exec = probe.exec_; + auto state = std::make_shared(std::move(probe)); + return JoinProbeFactory{[state](OperatorPtr child, bool preserve_empty) -> OperatorPtr { + return std::make_unique(std::move(child), *state, + preserve_empty); + }, + [state](OperatorPtr child, bool preserve_empty) -> OperatorPtr { + return std::make_unique( + std::move(child), std::move(*state), preserve_empty); + }, + exec}; +} + +/// The runtime value carried by the physical HashBuild -> HashProbe edge. +/// Orientation is represented by the variant alternative, so HashProbe never +/// re-decides which side was indexed. +struct StreamingHashProbeInput { + OperatorPtr source; + std::optional
materialized_source; + JoinProbe probe; +}; + +struct SwappedHashProbeInput { + Table left; + JoinProbe probe; +}; + +struct PrecomputedHashProbeInput { + Table output; +}; + +using HashProbeInput = + std::variant; + +/// Inner hash join for single-key no-predicate joins. +/// +/// Two execution modes: +/// - Stream: right is small (<= kStreamRightThreshold). Build a chained +/// hash index on the materialized right, then probe each left chunk +/// streamed from the child. Matches the classic star-join shape. +/// - Swapped: right is large and n_left < n_right. Materialize left, +/// build the hash index on left, iterate right rows once and emit output +/// in that same right-scan (probe) order (baseline's +/// `build_indices_from_right_scan` equivalent) — row order is outside the +/// join contract (SPEC.md §5.6), and preserving the probe side's scan order +/// instead of reassembling by left row keeps cache locality for any +/// downstream join that probes this join's output. Much better cache +/// behavior overall when the smaller side fits. +/// +/// Name conflicts are resolved with the same `_right` suffix rule as +/// `join_table_impl`. +class ChunkedInnerJoinOperator final : public Operator { + public: + ChunkedInnerJoinOperator(OperatorPtr left, Table right, const std::vector* keys, + const ExecutionContext& exec, ir::JoinSuffixPolicy suffix = {}, + const std::vector* pending_order = nullptr, + physical::JoinParallelism par = {}, + std::optional columns = std::nullopt) + : left_(std::move(left)), + right_(std::make_shared
(std::move(right))), + keys_(keys), + par_(par), + pending_order_(pending_order) { + bind_probe(keys, std::move(suffix), exec, std::move(columns)); + } + + /// Deferred-probe variant: the right side is an undecoded lazy scan (plus + /// its Project/Rename wrappers), interpreted only after this join has + /// published build-side key bounds into the scan's filter slot. The + /// registry/scalars/externs pointers are the interpret context and outlive + /// the operator. + ChunkedInnerJoinOperator(OperatorPtr left, const ir::Node* right_node, + const TableRegistry* registry, const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec, + const std::vector* keys, const DeferredScan* probe, + std::string probe_name, ir::JoinSuffixPolicy suffix = {}, + const std::vector* pending_order = nullptr, + physical::JoinParallelism par = {}, + std::optional columns = std::nullopt) + : left_(std::move(left)), + keys_(keys), + deferred_probe_(probe), + deferred_probe_name_(std::move(probe_name)), + deferred_right_node_(right_node), + deferred_registry_(registry), + deferred_scalars_(scalars), + deferred_externs_(externs), + deferred_exec_(&exec), + par_(par), + pending_order_(pending_order) { + bind_probe(keys, std::move(suffix), exec, std::move(columns)); + } + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (auto err = run_build()) { + return std::unexpected(std::move(*err)); + } + + if (mode_ == Mode::Precomputed) { + if (swapped_emitted_) { + return std::optional{}; + } + swapped_emitted_ = true; + if (precomputed_output_.rows() == 0) { + return std::optional{}; + } + return std::optional{table_to_chunk(std::move(precomputed_output_))}; + } + + if (mode_ == Mode::Swapped) { + if (swapped_emitted_) { + return std::optional{}; + } + swapped_emitted_ = true; + if (!left_table_.has_value()) { + return std::unexpected( + "ChunkedInnerJoinOperator: swapped mode without a materialized left table"); + } + auto out = probe_.emit_swapped(*left_table_); + if (!out.has_value()) { + return std::unexpected(std::move(out.error())); + } + if (out->rows() == 0) { + return std::optional{}; + } + return std::optional{table_to_chunk(std::move(*out))}; + } + + // Stream mode is the probe, and the probe is its own operator. The + // join constructs it on first use and delegates from here on: + // everything left in this class is build-side. + if (auto err = ensure_probe_op()) { + return std::unexpected(std::move(*err)); + } + return probe_op_->next(); + } + + /// Run this join's build phase to completion. + /// + /// The build is a phase with an explicit caller now, not a side effect of + /// whoever happens to pull the first chunk. `build_physical_join` runs it + /// at plan-execution time; `next()` still calls it, because a join reached + /// by any other path must work and because idempotence is what makes both + /// callers safe. After it returns the index is immutable and the probe + /// side can stream through it -- that is the barrier, stated as a call + /// rather than as a comment about `initialized_`. + /// + /// Deliberately NOT what this changes: it does not overlap the build with + /// anything. Overlapping a join's two sides has been tried twice and + /// regressed both times (`32889afd`, `27cb4a27`); this makes the build + /// schedulable, and what to schedule it against stays an open, measured + /// question. + auto run_build() -> std::optional { + if (initialized_) { + return std::nullopt; + } + if (auto err = initialize()) { + return err; + } + initialized_ = true; + return std::nullopt; + } + + /// Move the completed HashBuild result across the physical edge. This is + /// deliberately unavailable before `run_build`: HashProbe receives a + /// runtime-oriented value, not the mutable coordinator that produced it. + [[nodiscard]] auto take_hash_probe_input() -> std::expected { + if (!initialized_) { + return std::unexpected("physical HashBuild output requested before the build ran"); + } + if (mode_ == Mode::Precomputed) { + return HashProbeInput{ + PrecomputedHashProbeInput{.output = std::move(precomputed_output_)}}; + } + if (mode_ == Mode::Swapped) { + if (!left_table_.has_value()) { + return std::unexpected( + "ChunkedInnerJoinOperator: swapped mode without a materialized left table"); + } + SwappedHashProbeInput input{.left = std::move(*left_table_), + .probe = std::move(probe_)}; + left_table_.reset(); + return HashProbeInput{std::move(input)}; + } + + StreamingHashProbeInput input{.source = std::move(left_), + .materialized_source = std::move(probe_side_), + .probe = std::move(probe_)}; + probe_side_.reset(); + if (!input.materialized_source.has_value() && input.source == nullptr) { + return std::unexpected("physical HashProbe has no probe-side source"); + } + return HashProbeInput{std::move(input)}; + } + + private: + enum class Mode : std::uint8_t { Stream, Swapped, Precomputed }; + + /// The chain terminator, shared with the index this operator probes. + static constexpr std::size_t kNil = kJoinNil; + + // Build-on-right is preferred when right is small enough that probing + // it from streaming left chunks is cache-friendly. Above this, we + // materialize left to pick the smaller build side. + static constexpr std::size_t kStreamRightThreshold = 65536; + + /// Hand the probe what it needs before anything runs, including joint + /// ownership of the right side. The `shared_ptr`'s identity is stable from + /// construction even where the table it points at is filled later (the + /// deferred path fills it once the scan resolves), so this binds once and + /// the build phase writes through it. + void bind_probe(const std::vector* keys, ir::JoinSuffixPolicy suffix, + const ExecutionContext& exec, std::optional columns) { + probe_.keys_ = keys; + probe_.suffix_ = std::move(suffix); + probe_.exec_ = &exec; + probe_.right_ = right_; + probe_.probe_plan_ = par_.probe; + probe_.columns_ = std::move(columns); + } + + auto initialize() -> std::optional { + if (keys_->size() == 2) { + return initialize_pair(); + } + if (keys_->size() != 1) { + return "ChunkedInnerJoinOperator only supports single-key or two-Int64-key joins"; + } + if (deferred_probe_ != nullptr) { + if (auto err = resolve_deferred_probe()) { + return err; + } + if (mode_ == Mode::Precomputed) { + return std::nullopt; + } + } + const std::string& left_key_name = keys_->front().left; + const std::string& right_key_name = keys_->front().right; + const ColumnValue* rkey = right_->find(right_key_name); + if (rkey == nullptr) { + return "join key not found in right table: " + right_key_name; + } + ExprType key_kind = ExprType::Int; + if (auto err = detect_join_key_kind(*rkey, key_kind)) { + return err; + } + + const std::size_t n_right = right_->rows(); + + // Small right: index it without ever measuring the left, which is the + // one orientation this join can choose without draining a child. + if (n_right <= kStreamRightThreshold) { + return adopt_build(build_join_side(*right_, right_key_name, key_kind, + JoinOrientation::BuildRight, + build_partitions(n_right))); + } + + Table left_table; + if (use_materialized_left_ && left_materialized_.has_value()) { + // The deferred-probe path already drained the left child. + left_table = std::move(*left_materialized_); + left_materialized_.reset(); + use_materialized_left_ = false; + } else { + auto left_res = MaterializeOperator(std::move(left_)).run(); + if (!left_res.has_value()) { + return std::move(left_res.error()); + } + left_table = std::move(*left_res); + } + const std::size_t n_left = left_table.rows(); + + // Evaluated only when swapping was otherwise preferred: it is not a + // pure predicate (it can set up the probe's right-emit schema), so + // asking it unconditionally would move work the short-circuit used to + // skip. + const bool order_pays = + n_left < n_right && order_preserving_pays(left_table, n_left, n_right); + auto outcome = choose_and_build_single_key(left_table, *right_, left_key_name, + right_key_name, key_kind, order_pays, + build_partitions(std::min(n_left, n_right))); + return adopt_build(std::move(outcome), std::move(left_table)); + } + + /// How many partitions this join's hash build may fill concurrently, or 1 + /// to build it serially. + /// + /// The floor (`1U << 17U`) and the worker cap (`min(compute_budget, pool, + /// 64)`) used to be computed here; they are now the `hash-build` phase of + /// the physical plan (`physical::join_hash_build_parallelism`), resolved by + /// the builder and read from `par_.build`. src/runtime/PARALLELISM.md, + /// "Target: parallelism as a plan decision". The conditions that stay here + /// are the ones only the operator can judge: the `IBEX_JOIN_BUILD_SERIAL` + /// kill switch, whether it is nested under another fan-out + /// (`on_worker_pool_thread` -- no nested pool submissions), and whether + /// *this* build side cleared the floor (its row count is not known until + /// the side is materialized). The floor is higher than the probe's because + /// a partitioned build makes three passes over the keys where a serial + /// build makes one. + [[nodiscard]] auto build_partitions(std::size_t n) const -> std::size_t { + // Kill switch, and the A/B handle: with it set, the same binary runs + // the serial build, so the two can be interleaved without rebuilding. + if (std::getenv("IBEX_JOIN_BUILD_SERIAL") != nullptr) { + return 1; + } + if (par_.build.decline != physical::FanOutDecline::None || par_.build.worker_cap < 2 || + on_worker_pool_thread() || n < par_.build.row_floor) { + return 1; + } + // Telemetry only (see `parallel_hash_builds`): the fan-out is byte-identical + // to the serial build, so a gate that silently stopped matching would lose + // the parallelism with every test still green. Counted here rather than at + // the four call sites, which all route the result straight into the build. + if (probe_.exec_ != nullptr && probe_.exec_->parallel_stats != nullptr) { + probe_.exec_->parallel_stats->parallel_hash_builds.fetch_add(1, + std::memory_order_relaxed); + } + return par_.build.worker_cap; + } + + /// Publish what a build produced, and hand back the orientation it chose. + /// The single place `probe_.index_` is written: `probe_.index()` gives the + /// probe a `const` reference, so the barrier stays a compile error to + /// cross rather than a convention. + auto publish_build(std::expected outcome) + -> std::expected { + if (!outcome.has_value()) { + return std::unexpected(std::move(outcome.error())); + } + probe_.index_ = std::move(outcome->index); + return outcome->orientation; + } + + /// Apply what a build phase decided: publish the index and put this + /// operator into the shape that orientation implies. The build returns a + /// value; every member write that follows from it happens here and + /// nowhere else. + /// + /// `left_table` is the materialized left when the decision needed one -- + /// it becomes either the scanned-once swapped side or the single chunk + /// the stream path replays, depending on which way the build went. + auto adopt_build(std::expected outcome, + std::optional
left_table = std::nullopt) -> std::optional { + auto orientation = publish_build(std::move(outcome)); + if (!orientation.has_value()) { + return std::move(orientation.error()); + } + if (*orientation == JoinOrientation::BuildLeft) { + left_table_ = std::move(left_table); + mode_ = Mode::Swapped; + return std::nullopt; + } + // BuildRight: the other side streams through the index, which is what + // `JoinProbeOperator` does. A left that has already been drained -- + // either by the orientation decision above, or earlier by the + // deferred-probe path publishing its key bounds -- is replayed as one + // chunk rather than re-wrapped in its original operator, exactly as + // `use_materialized_left_` used to do inline. + // + // The second case is not hypothetical and cost a segfault to find: + // the deferred path drains `left_` to publish a filter, and if the + // resolved right then lands under `kStreamRightThreshold` the fast + // path arrives here with no `left_table` and a moved-from `left_`. + std::optional
materialized_probe_side; + if (left_table.has_value()) { + materialized_probe_side = std::move(*left_table); + } else if (use_materialized_left_ && left_materialized_.has_value()) { + materialized_probe_side = std::move(*left_materialized_); + left_materialized_.reset(); + use_materialized_left_ = false; + } + + // The probe side is kept rather than consumed. Building the probe + // operator here would settle a question a caller above may want to + // answer differently: a map pipeline over this join can take the probe + // and run it at the head of its own worker chains, which is the Umbra + // shape -- one build pipeline, then a probe pipeline whose maps run in + // the same worker as the probe. `take_fusible_probe` is that handoff, + // and it has to happen before `ensure_probe_op` commits. + probe_side_ = std::move(materialized_probe_side); + if (!probe_side_.has_value() && left_ == nullptr) { + // Every way of losing the probe side is a bug in the branches + // above, and one of them was. Aborting with a name beats + // dereferencing null inside a pool thread, which is what the + // deferred fast-path case actually did. + invariant_violation("join probe: no probe-side source after the build phase"); + } + return std::nullopt; + } + + /// Build Stream mode's probe operator, once, on first use. + /// + /// Split from `adopt_build` so a caller that wants to fuse this probe into + /// its own pipeline has a window in which the decision is still open -- + /// see `take_fusible_probe`. + auto ensure_probe_op() -> std::optional { + if (probe_op_ != nullptr) { + return std::nullopt; + } + auto built = make_join_probe_operator(std::move(left_), std::move(probe_side_), + make_probe_factory(std::move(probe_))); + if (!built.has_value()) { + return std::move(built.error()); + } + probe_side_.reset(); + probe_op_ = std::move(*built); + return std::nullopt; + } + + public: + /// Hand the probe and its input to a caller that will run them itself. + /// Empty unless this join settled on `BuildRight` and nothing has pulled + /// from it yet: a swapped or precomputed join emits one table and has no + /// probe pipeline to give. + /// + /// A probe side that is still streaming gets materialized here, which + /// sounds like a cost added and is not: the caller is a parallel map + /// pipeline, which was going to materialize the join's OUTPUT and + /// morselize that. This materializes the probe side instead, and the + /// join's output is then never assembled at all -- it is produced a morsel + /// at a time inside the workers. Which of the two tables is larger is a + /// real question and a measured one; it is not a question of whether a + /// materialization exists. + /// + /// The operator is left without a probe and must be discarded. + [[nodiscard]] auto take_fusible_probe() + -> std::expected, std::string> { + if (mode_ != Mode::Stream || probe_op_ != nullptr) { + return std::optional{}; + } + if (!probe_side_.has_value()) { + if (left_ == nullptr) { + return std::optional{}; + } + auto drained = MaterializeOperator(std::move(left_)).run(); + if (!drained.has_value()) { + return std::unexpected(std::move(drained.error())); + } + probe_side_ = std::move(*drained); + } + FusibleJoinProbe out{.probe_side = std::move(*probe_side_), + .probe = make_probe_factory(std::move(probe_))}; + probe_side_.reset(); + return std::optional{std::move(out)}; + } + + private: + /// Two-fixed-width-int-key path: narrow first cut of the streaming + /// two-key join (plans/parallelism-overview.md's "stream multi-key + /// joins" item). Non-deferred case: `right_` is already a whole `Table` + /// by construction (the call site materializes it, same as the + /// single-key path), so the only real decision left is which side to + /// index: this materializes `left_` too and builds on whichever side is + /// smaller -- the same motivation as `initialize()`'s single-key swap + /// decision, and necessary here because the call site cannot know in + /// advance which side a two-key join chain puts on which name (TPC-H + /// q09's `lineitem` join has the multi-million-row side as `right_`; + /// indexing it unconditionally was measured a >2x regression before this + /// fix). Deferred case: see `resolve_deferred_probe_pair`. + auto initialize_pair() -> std::optional { + if (deferred_probe_ != nullptr) { + return resolve_deferred_probe_pair(); + } + const ir::JoinKey& k0 = keys_->at(0); + const ir::JoinKey& k1 = keys_->at(1); + // Checked before the left child is drained, so an unusable right key + // still costs nothing to report. + if (auto right_keys = pair_key_columns(*right_, k0.right, k1.right, "right"); + !right_keys.has_value()) { + return std::move(right_keys.error()); + } + Table left_table; + if (use_materialized_left_ && left_materialized_.has_value()) { + // `resolve_deferred_probe_pair` already drained the left child + // while deciding whether a scan filter was worth publishing. + left_table = std::move(*left_materialized_); + left_materialized_.reset(); + use_materialized_left_ = false; + } else { + auto left_res = MaterializeOperator(std::move(left_)).run(); + if (!left_res.has_value()) { + return std::move(left_res.error()); + } + left_table = std::move(*left_res); + } + probe_.pair_mode_ = true; + // On the BuildRight side of this decision the left is already fully + // materialized, so `adopt_build` drains it as a single chunk through + // the existing `use_materialized_left_` mechanism rather than + // re-wrapping it in an operator. + auto outcome = + choose_and_build_pair(left_table, *right_, k0, k1, + build_partitions(std::min(left_table.rows(), right_->rows()))); + return adopt_build(std::move(outcome), std::move(left_table)); + } + + /// Deferred-probe POC for the two-key path (plans/parallelism-overview.md + /// "deferred scan filtering for two-key joins", TPC-H q09's `lineitem` + /// join). Reuses the existing single-key deferred-scan machinery + /// unchanged: builds the (small) left side first, publishes a + /// `DynamicScanFilter` over `keys_->at(0)` ONLY -- one component, not + /// both -- into the scan's filter slot, then lets the source's normal + /// decode-time pruning narrow the right side before it is ever + /// materialized. Membership in one component is necessary but not + /// sufficient for the pair match, so this can only produce harmless + /// false positives (rows sharing q09's l_partkey but not l_suppkey); + /// `choose_and_build_pair`'s exact pair probe afterward is what + /// actually enforces both keys, unchanged from the non-deferred path. + /// + /// Same `kStreamRightThreshold` gate as the single-key + /// `resolve_deferred_probe`: below it, publishing a filter (a Bloom plus + /// a sort/unique pass over the whole build side) can only add cost, not + /// recover it, since the deferred side was never going to be expensive to + /// decode in the first place. First cut of this POC always materialized + /// left and published a filter regardless of size -- measured a clean, + /// unanimous +8.4% regression on q05 (its `join supplier on + /// {l_suppkey=s_suppkey, n_nationkey=s_nationkey}` is exactly this + /// shape, `supplier` fitting in one row group with nothing to prune). + /// Below the threshold this falls through to `initialize_pair`'s + /// ordinary side-picking, reusing the already-drained left side via + /// `left_materialized_`/`use_materialized_left_` rather than draining it + /// twice. + /// + /// No two-phase probe (`try_two_phase_probe`'s candidate-selection + /// optimization) here -- that is a further, separable lever on top of + /// scan-altitude pruning, and this POC is scoped to answering whether + /// pruning the scan itself is worth it at all before adding more on top. + auto resolve_deferred_probe_pair() -> std::optional { + DynamicScanFilter& slot = *deferred_probe_->filter; + if (deferred_probe_->lazy->rows() > kStreamRightThreshold) { + auto left_res = MaterializeOperator(std::move(left_)).run(); + if (!left_res.has_value()) { + return std::move(left_res.error()); + } + publish_build_filter_column(*left_res, keys_->at(0).left, slot); + left_materialized_ = std::move(*left_res); + use_materialized_left_ = true; + } + slot.ready = true; + auto right = interpret_node(*deferred_right_node_, *deferred_registry_, deferred_scalars_, + deferred_externs_, *deferred_exec_); + if (!right.has_value()) { + return std::move(right.error()); + } + *right_ = std::move(*right); + deferred_probe_ = nullptr; + if (std::getenv("IBEX_DEBUG_PAIR_DEFER") != nullptr) { + ibex::formatting::print(stderr, + "[pair_defer] filter_published={} right_rows_after_filter={}\n", + static_cast(use_materialized_left_), right_->rows()); + } + return initialize_pair(); + } + + /// The probe side is an undecoded lazy scan. When it is worth it, drain + /// the build (left) side first and publish its key filter (membership + + /// bounds) into the scan's filter slot, so the scan skips materializing + /// rows that cannot match. Every path marks the slot ready before the + /// scan is interpreted; the filter is an optimization the slot may + /// simply not carry. + auto resolve_deferred_probe() -> std::optional { + DynamicScanFilter& slot = *deferred_probe_->filter; + // Pre-filter row count: an upper bound on the decoded size, good + // enough to decide whether the probe side is large enough to bother. + if (deferred_probe_->lazy->rows() > kStreamRightThreshold) { + auto left_res = MaterializeOperator(std::move(left_)).run(); + if (!left_res.has_value()) { + return std::move(left_res.error()); + } + publish_build_filter(*left_res, slot); + left_materialized_ = std::move(*left_res); + use_materialized_left_ = true; + } + slot.ready = true; + if (use_materialized_left_) { + TwoPhase outcome = TwoPhase::NotApplicable; + if (auto err = try_two_phase_probe(slot, outcome)) { + return err; + } + if (outcome == TwoPhase::Precomputed) { + deferred_probe_ = nullptr; + mode_ = Mode::Precomputed; + return std::nullopt; + } + if (outcome == TwoPhase::RightMaterialized) { + // Phase A ran but full two-phase declined; its selection was + // reused to materialize right_, so fall through to the + // ordinary side-picking in initialize(). + deferred_probe_ = nullptr; + return std::nullopt; + } + } + auto right = interpret_node(*deferred_right_node_, *deferred_registry_, deferred_scalars_, + deferred_externs_, *deferred_exec_); + if (!right.has_value()) { + return std::move(right.error()); + } + *right_ = std::move(right.value()); + deferred_probe_ = nullptr; + return std::nullopt; + } + + enum class TwoPhase : std::uint8_t { NotApplicable, RightMaterialized, Precomputed }; + + /// Interpret the Project/Rename/Update wrappers over an already + /// materialized scan table by shadowing the scan name in a registry + /// copy — the Scan case hits the registry before the deferred fallback. + auto interpret_wrapped_right(Table scan_table) -> std::optional { + TableRegistry local = *deferred_registry_; + local.insert_or_assign(deferred_probe_name_, std::move(scan_table)); + auto right = interpret_node(*deferred_right_node_, local, deferred_scalars_, + deferred_externs_, *deferred_exec_); + if (!right.has_value()) { + return std::move(right.error()); + } + *right_ = std::move(right.value()); + return std::nullopt; + } + + /// Late materialization across the join (decode-fusion stage 5): probe + /// with just the scan's key column, then decode the payload columns only + /// for the rows that actually matched. When every survivor matched + /// exactly one build row (unique build keys — the common star shape), + /// the probe-side columns pass into the output without a gather. + /// + /// NotApplicable (nothing ran — no membership filter, or phase A had no + /// selective answer): the caller interprets the subtree as before. When + /// phase A DID run but full two-phase declines — the build side is + /// larger than the candidate set (two-phase forces build-on-left; the + /// ordinary side-picking may do better) or a key type surprise — its + /// selection is reused to materialize `right_` (RightMaterialized) + /// rather than thrown away: recomputing it from scratch was measured at + /// +12% on q03. + auto try_two_phase_probe(const DynamicScanFilter& slot, TwoPhase& outcome) + -> std::optional { + outcome = TwoPhase::NotApplicable; + if (!slot.has_membership() || !left_materialized_.has_value()) { + return std::nullopt; + } + const Table& build = *left_materialized_; + const auto* build_entry = build.find_entry(keys_->front().left); + if (build_entry == nullptr || + !std::holds_alternative>(*build_entry->column)) { + return std::nullopt; + } + + auto phase = deferred_scan_key_selection(*deferred_probe_, *deferred_exec_); + if (!phase.has_value()) { + return std::move(phase.error()); + } + if (!phase->has_value()) { + return std::nullopt; + } + auto sel = std::move(**phase); + const auto* keys_col = std::get_if>(&*sel.keys.column); + if (build.rows() > sel.selected.size() || keys_col == nullptr) { + auto right_rows = materialize_deferred_scan_rows(*deferred_probe_, sel.selected, + *deferred_exec_, std::move(sel.keys)); + if (!right_rows.has_value()) { + return std::move(right_rows.error()); + } + if (auto err = interpret_wrapped_right(std::move(*right_rows))) { + return err; + } + outcome = TwoPhase::RightMaterialized; + return std::nullopt; + } + + // Publish only: this path builds on the left and scans the right, but + // it emits one precomputed table rather than running either streaming + // shape, so it takes no operator mode from the orientation. + if (auto published = publish_build( + build_join_side(build, keys_->front().left, ExprType::Int, + JoinOrientation::BuildLeft, build_partitions(build.rows()))); + !published.has_value()) { + return std::move(published.error()); + } + + // Probe the candidate keys in scan order; record one hit per + // surviving row plus the expanded (build row, survivor) pairs — the + // same probe-order-major layout emit_swapped produces. + const auto* key_data = keys_col->data(); + const ValidityBitmap* key_validity = + sel.keys.validity.has_value() ? &*sel.keys.validity : nullptr; + const std::size_t n = keys_col->size(); + + // Same scan/replay shape as `probe_swapped`, with a twist: `ri` here + // indexes HITS (the survivor list), not probe rows, so each part + // needs two prefix offsets — its first hit index and its first output + // pair — before the replays can write disjoint slices. One part when + // the gate declines, so the serial path is the same code. + const auto scan = [&](std::size_t begin, std::size_t end, + std::vector& hits, std::size_t& total) { + for (std::size_t i = begin; i < end; ++i) { + if (key_validity != nullptr && !(*key_validity)[i]) { + continue; + } + const std::size_t head = probe_.index().i64_heads.find_head(key_data[i]); + if (head == kNil) { + continue; + } + hits.push_back(JoinProbe::SwappedHit{.rrow = i, .head = head}); + for (std::size_t cur = head; cur != kNil; cur = probe_.index().chain_next[cur]) { + ++total; + } + } + }; + const std::size_t workers = probe_.probe_parallel_workers(n); + if (workers == 0) { + probe_.swapped_parts_.resize(1); + probe_.swapped_parts_[0].hits.clear(); + probe_.swapped_parts_[0].total = 0; + scan(0, n, probe_.swapped_parts_[0].hits, probe_.swapped_parts_[0].total); + } else { + auto& pool = process_worker_pool(); + const std::size_t grain = (n + workers - 1) / workers; + probe_.swapped_parts_.resize(workers); + auto batch = pool.submit(workers, [&](std::size_t w) { + auto& part = probe_.swapped_parts_[w]; + part.hits.clear(); + part.total = 0; + const std::size_t begin = w * grain; + const std::size_t end = std::min(n, begin + grain); + if (begin < end) { + scan(begin, end, part.hits, part.total); + } + }); + batch.wait(); + } + const std::size_t n_parts = probe_.swapped_parts_.size(); + std::vector hit_offsets(n_parts); + probe_.part_offsets_.resize(n_parts); + std::size_t n_hits = 0; + std::size_t total = 0; + for (std::size_t w = 0; w < n_parts; ++w) { + hit_offsets[w] = n_hits; + probe_.part_offsets_[w] = total; + n_hits += probe_.swapped_parts_[w].hits.size(); + total += probe_.swapped_parts_[w].total; + } + + Selection survivors(n_hits); + std::vector li(total, 0); + std::vector ri(total, 0); + Column gathered_keys; + const bool gather_keys = n_hits != n; + if (gather_keys) { + gathered_keys.resize_for_overwrite(n_hits); + } + // Detach once here, not per element inside the replay: the mutable + // `operator[]` pays a CoW check every call, and on a worker the + // detach itself would race. + std::int64_t* gathered_out = gather_keys ? gathered_keys.data() : nullptr; + const auto replay = [&](std::size_t w) { + const auto& part = probe_.swapped_parts_[w]; + std::size_t h = hit_offsets[w]; + std::size_t pos = probe_.part_offsets_[w]; + for (const JoinProbe::SwappedHit& hit : part.hits) { + survivors[h] = sel.selected[hit.rrow]; + if (gathered_out != nullptr) { + gathered_out[h] = key_data[hit.rrow]; + } + for (std::size_t cur = hit.head; cur != kNil; + cur = probe_.index().chain_next[cur]) { + li[pos] = cur; + ri[pos] = h; + ++pos; + } + ++h; + } + }; + if (workers == 0) { + replay(0); + } else { + auto batch = process_worker_pool().submit(n_parts, replay); + batch.wait(); + if (deferred_exec_->parallel_stats != nullptr) { + deferred_exec_->parallel_stats->parallel_probes.fetch_add( + 1, std::memory_order_relaxed); + } + } + const bool ri_identity = total == n_hits; + + // Survivors' key values, gathered in memory from phase A's keys. + ColumnEntry key_entry; + key_entry.name = sel.keys.name; + if (!gather_keys) { + key_entry.column = sel.keys.column; + key_entry.validity = sel.keys.validity; + } else { + key_entry.column = std::make_shared(std::move(gathered_keys)); + // Null keys never match, so every survivor's key is valid. + } + + auto right_rows = materialize_deferred_scan_rows(*deferred_probe_, survivors, + *deferred_exec_, std::move(key_entry)); + if (!right_rows.has_value()) { + return std::move(right_rows.error()); + } + if (auto err = interpret_wrapped_right(std::move(*right_rows))) { + return err; + } + + Table left_copy; + left_copy.columns.reserve(build.columns.size()); + for (const auto& c : build.columns) { + left_copy.add_column(c.name, *c.column); + left_copy.columns.back().validity = c.validity; + } + auto out = probe_.assemble_output(std::move(left_copy), li.data(), ri.data(), total, + /*li_identity=*/false, ri_identity); + if (!out.has_value()) { + return std::move(out.error()); + } + precomputed_output_ = std::move(*out); + outcome = TwoPhase::Precomputed; + return std::nullopt; + } + + // Derive the probe scan's dynamic filter from the build side's valid key + // values (int keys only; other key types publish nothing). Sound for any + // inner join regardless of which side ends up as the build: a probe row + // whose key fails the filter cannot match. + // + // Everything here is published ungated — membership because a range + // estimate cannot predict set selectivity (the scan decides with a + // sampled pass rate), and min/max because the consumer owns the policy: + // materialize_deferred_scan gates conjunct synthesis on estimated + // pruning, the fused key scan uses the raw bounds for row-group + // skipping. + void publish_build_filter(const Table& build, DynamicScanFilter& slot) const { + publish_build_filter_column(build, keys_->front().left, slot); + } + + // Component-selecting variant for the two-key deferred-probe POC + // (`resolve_deferred_probe_pair`): publishes a filter over exactly one + // named build-side column instead of always `keys_->front().left`, since + // the pair join's scan filter only ever covers one of the two keys. + static void publish_build_filter_column(const Table& build, const std::string& key_name, + DynamicScanFilter& slot) { + const auto* entry = build.find_entry(key_name); + if (entry == nullptr) { + return; + } + const auto* col = std::get_if>(&*entry->column); + if (col == nullptr || col->empty()) { + return; + } + const ValidityBitmap* validity = entry->validity.has_value() ? &*entry->validity : nullptr; + const auto* data = col->data(); + const std::size_t n = col->size(); + std::int64_t mn = std::numeric_limits::max(); + std::int64_t mx = std::numeric_limits::min(); + std::size_t valid_rows = 0; + for (std::size_t r = 0; r < n; ++r) { + if (validity != nullptr && !(*validity)[r]) { + continue; + } + mn = std::min(mn, data[r]); + mx = std::max(mx, data[r]); + ++valid_rows; + } + if (valid_rows == 0) { + return; + } + + // Every build side gets a Bloom — even alongside an exact list, the + // Bloom is the probe fast path (see DynamicScanFilter::passes). + // Duplicate inserts are harmless. A small build side (dimension + // chains: nation, region, filtered part) additionally dedups cheaply + // into an exact list, cancelling the Bloom's false positives. + constexpr std::size_t kInListBuildMax = 4096; + constexpr std::size_t kInListMax = 1024; + JoinBloomFilter bloom(valid_rows); + for (std::size_t r = 0; r < n; ++r) { + if (validity != nullptr && !(*validity)[r]) { + continue; + } + bloom.insert(data[r]); + } + slot.bloom = std::move(bloom); + if (valid_rows <= kInListBuildMax) { + std::vector keys; + keys.reserve(valid_rows); + for (std::size_t r = 0; r < n; ++r) { + if (validity != nullptr && !(*validity)[r]) { + continue; + } + keys.push_back(data[r]); + } + std::ranges::sort(keys); + keys.erase(std::ranges::unique(keys).begin(), keys.end()); + if (keys.size() <= kInListMax) { + slot.in_list = std::move(keys); + } + } + // Raw facts, not policy: whether these bounds are worth acting on is + // the consumer's call — materialize_deferred_scan gates synthesized + // conjuncts on estimated pruning, while the fused key scan uses them + // ungated to skip whole row groups (which has no gather downside). + slot.min = mn; + slot.max = mx; + } + + /// Would indexing the right instead of the left buy the pending `order`, + /// and is the index small enough that it is worth buying? + /// + /// The pending keys are in the join's output names and the left's claim is + /// in the left's own, so the claim is restated through the output plan + /// before they are compared -- a suffixed key is renamed and a key the + /// output drops takes the claim with it. + auto order_preserving_pays(const Table& left_table, std::size_t n_left, std::size_t n_right) + -> bool { + constexpr std::size_t kMaxOrderPreservingBuildRatio = 4; + const auto& left_ordering = left_table.properties().ordering(); + if (pending_order_ == nullptr || pending_order_->empty() || !left_ordering.has_value() || + n_right > kMaxOrderPreservingBuildRatio * n_left) { + return false; + } + if (!probe_.right_emit_ready_) { + if (auto ready = probe_.setup_right_emit_schema(left_table); !ready.has_value()) { + return false; // the join is about to fail on this anyway + } + } + std::vector carried; + for (const auto& key : *left_ordering) { + std::size_t idx = left_table.columns.size(); + for (std::size_t i = 0; i < left_table.columns.size(); ++i) { + if (left_table.columns[i].name == key.name) { + idx = i; + break; + } + } + if (idx == left_table.columns.size() || idx >= probe_.left_emit_names_.size()) { + return false; + } + carried.push_back( + ir::OrderKey{.name = probe_.left_emit_names_[idx], .ascending = key.ascending}); + } + return TableProperties::sorted_by(std::move(carried)).satisfies(*pending_order_); + } + + OperatorPtr left_; + /// The right side, owned jointly with every probe reading it. A + /// `shared_ptr` rather than a member `Table` because a probe must be able + /// to outlive this operator and several probes must be able to read one + /// build side at once -- see `JoinProbe::right_`. Written only by the + /// build phase, which finishes before any probe runs. + std::shared_ptr
right_ = std::make_shared
(); + const std::vector* keys_; + /// The probe half. The operator runs the build and decides which side to + /// index; everything after that belongs here. Moved into `probe_op_` when + /// the orientation is BuildRight, since from then on the probe is an + /// operator of its own and this class is build-side only. + JoinProbe probe_; + /// The probe side, drained by the build phase and held until either + /// `ensure_probe_op` turns it into a source or `take_fusible_probe` hands + /// it to a pipeline above. Empty when the probe side streams from `left_`. + std::optional
probe_side_; + /// Stream mode's probe, constructed on first use: one + /// `JoinProbeOperator` over the probe-side child, or -- when that child + /// was already materialized and is big enough -- a morsel pipeline of + /// several of them over its morsels. Null in the Swapped and Precomputed + /// modes, which emit one table rather than streaming. + OperatorPtr probe_op_; + + // Deferred-probe context (see the second constructor). `deferred_probe_` + // doubles as the mode flag: non-null until the probe scan is resolved. + const DeferredScan* deferred_probe_ = nullptr; + std::string deferred_probe_name_; + const ir::Node* deferred_right_node_ = nullptr; + const TableRegistry* deferred_registry_ = nullptr; + const ScalarRegistry* deferred_scalars_ = nullptr; + const ExternRegistry* deferred_externs_ = nullptr; + const ExecutionContext* deferred_exec_ = nullptr; + bool initialized_ = false; + Mode mode_ = Mode::Stream; + + /// Both fan-out phases' parallelism, resolved by the caller + /// (`resolved_join_parallelism`, shared by every construction site). + /// `build_partitions` reads `par_.build`; `bind_probe` copies `par_.probe` + /// into the probe. src/runtime/PARALLELISM.md. Default-constructed + /// (`worker_cap == 0`) only on a hand-built operator, where both phases then + /// stay serial. + physical::JoinParallelism par_{}; + + // What an `order` above this join will ask for, or null. Only ever shifts + // which side is indexed; see `initialize`. + const std::vector* pending_order_ = nullptr; + + // Stream mode: when right > threshold and left >= right, left was + // materialized to measure but not swapped; replay as a single chunk. + std::optional
left_materialized_; + bool use_materialized_left_ = false; + std::optional
empty_schema_; + + // Swapped mode: materialized left held for later gather. + std::optional
left_table_; + bool swapped_emitted_ = false; + + // Precomputed mode: the two-phase deferred probe assembled the whole + // join output during initialization. + Table precomputed_output_; +}; + +/// HashProbe for the runtime BuildLeft orientation. The build has already +/// produced the immutable index and retained the materialized left side; this +/// operator only scans the right side through that index and emits once. +class SwappedHashProbeOperator final : public Operator { + public: + explicit SwappedHashProbeOperator(SwappedHashProbeInput input) + : left_(std::move(input.left)), probe_(std::move(input.probe)) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (emitted_) { + return std::optional{}; + } + emitted_ = true; + auto out = probe_.emit_swapped(left_); + if (!out.has_value()) { + return std::unexpected(std::move(out.error())); + } + if (out->rows() == 0) { + return std::optional{}; + } + return std::optional{table_to_chunk(std::move(*out))}; + } + + private: + Table left_; + JoinProbe probe_; + bool emitted_ = false; +}; + +/// Deferred joins can finish during HashBuild after their dynamic filter has +/// resolved the probe source. They still cross the same typed edge; HashProbe +/// simply emits the already-computed result rather than re-running work. +class PrecomputedHashProbeOperator final : public Operator { + public: + explicit PrecomputedHashProbeOperator(Table output) : output_(std::move(output)) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (emitted_) { + return std::optional{}; + } + emitted_ = true; + if (output_.rows() == 0) { + return std::optional{}; + } + return std::optional{table_to_chunk(std::move(output_))}; + } + + private: + Table output_; + bool emitted_ = false; +}; + +/// Construct the physical HashProbe from exactly one completed HashBuild +/// output. There is no orientation branch after this point: the variant chosen +/// by the build owns the only legal probe implementation for that orientation. +auto build_hash_probe_operator(HashProbeInput input) -> std::expected { + if (auto* stream = std::get_if(&input)) { + return make_join_probe_operator(std::move(stream->source), + std::move(stream->materialized_source), + make_probe_factory(std::move(stream->probe))); + } + if (auto* swapped = std::get_if(&input)) { + return OperatorPtr{std::make_unique(std::move(*swapped))}; + } + auto& precomputed = std::get(input); + return OperatorPtr{ + std::make_unique(std::move(precomputed.output))}; +} + +} // namespace + +auto deferred_probe_scan_of(const ir::Node& right, const ExecutionContext& exec) + -> DeferredProbeScan { + return deferred_probe_scan_impl(right, exec); +} + +auto make_chunked_inner_join_operator(OperatorPtr left, Table right, + const std::vector* keys, + const ExecutionContext& exec, ir::JoinSuffixPolicy suffix, + const std::vector* pending_order, + physical::JoinParallelism parallelism, + std::optional columns) -> OperatorPtr { + return std::make_unique(std::move(left), std::move(right), keys, exec, + std::move(suffix), pending_order, parallelism, + std::move(columns)); +} + +namespace { + +auto finish_scheduled_join(std::unique_ptr op) + -> std::expected { + static const bool lazy = std::getenv("IBEX_JOIN_BUILD_LAZY") != nullptr; + if (lazy) { + return OperatorPtr{std::move(op)}; + } + if (auto err = op->run_build()) { + return std::unexpected(std::move(*err)); + } + auto probe_input = op->take_hash_probe_input(); + if (!probe_input.has_value()) { + return std::unexpected(std::move(probe_input.error())); + } + return build_hash_probe_operator(std::move(*probe_input)); +} + +} // namespace + +auto make_scheduled_chunked_inner_join_operator( + OperatorPtr left, Table right, const std::vector* keys, + const ExecutionContext& exec, ir::JoinSuffixPolicy suffix, + const std::vector* pending_order, physical::JoinParallelism parallelism, + std::optional columns) -> std::expected { + return finish_scheduled_join(std::make_unique( + std::move(left), std::move(right), keys, exec, std::move(suffix), pending_order, + parallelism, std::move(columns))); +} + +auto make_scheduled_deferred_inner_join_operator( + OperatorPtr left, const ir::Node* right_node, const TableRegistry* registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, const ExecutionContext& exec, + const std::vector* keys, const DeferredScan* probe, std::string probe_name, + ir::JoinSuffixPolicy suffix, const std::vector* pending_order, + physical::JoinParallelism parallelism, std::optional columns) + -> std::expected { + return finish_scheduled_join(std::make_unique( + std::move(left), right_node, registry, scalars, externs, exec, keys, probe, + std::move(probe_name), std::move(suffix), pending_order, parallelism, std::move(columns))); +} + +auto take_fusible_join_probe(OperatorPtr left, Table right, const std::vector* keys, + const ExecutionContext& exec, ir::JoinSuffixPolicy suffix, + const std::vector* pending_order, + physical::JoinParallelism parallelism) + -> std::expected, std::string> { + auto op = + std::make_unique(std::move(left), std::move(right), keys, exec, + std::move(suffix), pending_order, parallelism); + if (auto err = op->run_build()) { + return std::unexpected(std::move(*err)); + } + return op->take_fusible_probe(); +} + +} // namespace ibex::runtime diff --git a/src/runtime/join_chunked_internal.hpp b/src/runtime/join_chunked_internal.hpp new file mode 100644 index 00000000..4aa36dd1 --- /dev/null +++ b/src/runtime/join_chunked_internal.hpp @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "physical_plan.hpp" + +namespace ibex::runtime { + +/// Copyable construction handle for one worker-private streaming join probe. +/// Attaching normally clones the pristine probe state for a worker; the +/// single-consumer path moves it directly into its operator. Both retain +/// shared ownership of the immutable hash build. +class JoinProbeFactory { + public: + using Attach = std::function; + + JoinProbeFactory() = default; + JoinProbeFactory(Attach attach, Attach attach_move, const ExecutionContext* exec) + : attach_(std::move(attach)), attach_move_(std::move(attach_move)), exec_(exec) {} + + [[nodiscard]] auto attach(OperatorPtr child, bool preserve_empty_morsels = false) const + -> OperatorPtr { + return attach_(std::move(child), preserve_empty_morsels); + } + [[nodiscard]] auto attach_move(OperatorPtr child, bool preserve_empty_morsels = false) + -> OperatorPtr { + return attach_move_(std::move(child), preserve_empty_morsels); + } + [[nodiscard]] auto execution_context() const noexcept -> const ExecutionContext* { + return exec_; + } + + private: + Attach attach_; + Attach attach_move_; + const ExecutionContext* exec_ = nullptr; +}; + +struct FusibleJoinProbe { + Table probe_side; + JoinProbeFactory probe; +}; + +struct DeferredProbeScan { + const DeferredScan* scan = nullptr; + const std::string* name = nullptr; +}; + +[[nodiscard]] auto deferred_probe_scan_of(const ir::Node& right, const ExecutionContext& exec) + -> DeferredProbeScan; + +/// Pipeline-owned adapter supplied by chunked.cpp. It may morselize an already +/// materialized probe side before attaching worker-private probes. +[[nodiscard]] auto make_join_probe_operator(OperatorPtr source, + std::optional
materialized_source, + JoinProbeFactory probe) + -> std::expected; + +[[nodiscard]] auto make_chunked_inner_join_operator( + OperatorPtr left, Table right, const std::vector* keys, + const ExecutionContext& exec, ir::JoinSuffixPolicy suffix = {}, + const std::vector* pending_order = nullptr, + physical::JoinParallelism parallelism = {}, + std::optional columns = std::nullopt) -> OperatorPtr; + +[[nodiscard]] auto make_scheduled_chunked_inner_join_operator( + OperatorPtr left, Table right, const std::vector* keys, + const ExecutionContext& exec, ir::JoinSuffixPolicy suffix = {}, + const std::vector* pending_order = nullptr, + physical::JoinParallelism parallelism = {}, + std::optional columns = std::nullopt) + -> std::expected; + +[[nodiscard]] auto make_scheduled_deferred_inner_join_operator( + OperatorPtr left, const ir::Node* right_node, const TableRegistry* registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, const ExecutionContext& exec, + const std::vector* keys, const DeferredScan* probe, std::string probe_name, + ir::JoinSuffixPolicy suffix = {}, const std::vector* pending_order = nullptr, + physical::JoinParallelism parallelism = {}, + std::optional columns = std::nullopt) + -> std::expected; + +[[nodiscard]] auto take_fusible_join_probe(OperatorPtr left, Table right, + const std::vector* keys, + const ExecutionContext& exec, + ir::JoinSuffixPolicy suffix = {}, + const std::vector* pending_order = nullptr, + physical::JoinParallelism parallelism = {}) + -> std::expected, std::string>; + +} // namespace ibex::runtime diff --git a/src/runtime/packed_key_encoder_internal.hpp b/src/runtime/packed_key_encoder_internal.hpp new file mode 100644 index 00000000..dda153b4 --- /dev/null +++ b/src/runtime/packed_key_encoder_internal.hpp @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ibex::runtime { + +struct PackedKeyEncoder { + // MSVC has no __uint128_t. This is only a packed identity key, so an array + // of words is both portable and avoids pulling a compiler-specific integer + // type into the packed key path. + template + struct PackedWords { + std::array w{}; + + [[nodiscard]] friend auto operator==(const PackedWords&, const PackedWords&) + -> bool = default; + }; + template + struct PackedWordsHash { + auto operator()(const PackedWords& value) const noexcept -> std::size_t { + std::uint64_t acc = 0; + for (const auto word : value.w) { + acc ^= word + 0x9e3779b97f4a7c15ULL + (acc << 6U) + (acc >> 2U); + } + return static_cast(acc); + } + }; + using Packed128 = PackedWords<2>; + using Packed256 = PackedWords<4>; + + /// OR `cell` into the packed key at bit offset `shift`. A cell never spans + /// more than two words because no cell is wider than 64 bits. + template + static void splice(PackedWords& key, std::uint64_t cell, unsigned shift) { + const unsigned word = shift / 64U; + const unsigned off = shift % 64U; + key.w[word] |= cell << off; + // `cell >> 64` is UB, so the carry into the next word is only taken when + // the cell actually straddles the boundary. + if (off != 0 && word + 1 < Words) { + key.w[word + 1] |= cell >> (64U - off); + } + } + + /// The exact inverse of `splice`/the single-word shift in `pack_row`: + /// recover the `width_bits`-wide cell that was spliced in at bit offset + /// `shift`. Only ever needed to decode an ALREADY-PACKED key back into its + /// per-column values (fast-path migration); packing itself never reads a + /// cell back out, so this has no hot-path cost. + template + [[nodiscard]] static auto extract_cell(const Packed& key, unsigned shift, unsigned width_bits) + -> std::uint64_t { + const std::uint64_t mask = + width_bits >= 64U ? ~std::uint64_t{0} : ((std::uint64_t{1} << width_bits) - 1); + if constexpr (std::is_same_v) { + return (key >> shift) & mask; + } else { + const unsigned word = shift / 64U; + const unsigned off = shift % 64U; + std::uint64_t value = key.w[word] >> off; + if (off != 0 && word + 1 < key.w.size()) { + value |= key.w[word + 1] << (64U - off); + } + return value & mask; + } + } + + /// One fixed-width integral key column, resolved to its raw storage and the + /// bit offset it occupies in the packed key. + struct PackCol { + enum class Kind : std::uint8_t { Int64, Date, Ts, Bool, Cat } kind{Kind::Int64}; + const std::int64_t* i64 = nullptr; + const Date* date = nullptr; + const Timestamp* ts = nullptr; + const Column* boolean = nullptr; + const Column* cat = nullptr; + const std::uint32_t* remap = nullptr; ///< local code -> operator-global id + unsigned shift = 0; ///< bit offset of this column's cell in the packed key + }; + struct PackedPlan { + std::vector cols; + unsigned width = 0; ///< total packed width in bytes + }; + + /// Bit width of one packed cell, matching the byte counts + /// `build_packed_layout` accumulates per `PackCol::Kind`. + [[nodiscard]] static auto width_bits_of(PackCol::Kind kind) -> unsigned { + switch (kind) { + case PackCol::Kind::Int64: + case PackCol::Kind::Ts: + return 64U; + case PackCol::Kind::Date: + case PackCol::Kind::Cat: + return 32U; + case PackCol::Kind::Bool: + return 8U; + } + return 0U; + } + + /// Per-key-column interning state for Categorical columns. + /// + /// A categorical code is only meaningful against ITS OWN chunk's dictionary, + /// so packing the raw code would merge two different values that happen to + /// share a code in different chunks. Resolving each dictionary entry to an + /// operator-global id fixes that, and costs one lookup per DICTIONARY ENTRY + /// per chunk rather than one per row: the row loop then reads `remap[code]`, + /// a single array index with no hashing and no allocation at all. + struct CatIntern { + /// Views point into `arena`, whose deque never invalidates references. + robin_hood::unordered_flat_map ids; + std::deque arena; + std::vector remap; ///< rebuilt per chunk, indexed by local code + }; + + /// A key is packable iff every column reduces to a fixed-width INTEGRAL cell + /// whose byte equality equals value equality, with no nulls, and the columns + /// together fit in 32 bytes. + /// + /// Doubles are excluded (-0.0/NaN break byte equality). Strings are excluded + /// because interning one per row would cost the hash lookup this path exists + /// to avoid. Categoricals ARE included: their dictionary is interned once per + /// chunk into operator-global ids (see `CatIntern`), which is what makes a + /// code comparable across chunks. + auto build_packed_key(const std::vector& entries) + -> std::optional { + for (const auto* entry : entries) { + if (entry->validity.has_value()) { + return std::nullopt; + } + } + return build_packed_layout(entries); + } + + /// Same as `build_packed_key`, minus the "no column may carry nulls" check. + /// + /// Used to recover a stable fast path's (kind, shift) layout when a LATER + /// chunk's nulls are exactly what disqualifies `build_packed_key` -- the + /// layout itself does not depend on nullability, only on each column's + /// type and position, which stay fixed for the life of the query once the + /// packed path has been selected. The migration path calls this to learn + /// how to decode the packed keys a prior, null-free chunk already built. + auto build_packed_layout(const std::vector& entries) + -> std::optional { + // Size the interning state ONCE, before any of it is pointed at. + // + // `intern_categorical` hands back `remap.data()`, and `PackCol` holds + // that pointer for the rest of the chunk. Growing `cat_interns_` while + // those pointers are live reallocates the vector, and `CatIntern` holds + // a robin_hood map whose move constructor is not noexcept — so + // `move_if_noexcept` COPIES, `remap` gets a fresh buffer, and column + // 0's pointer is left dangling the moment column 1 is interned. + // + // The symptom was a second categorical key column silently reading + // freed memory: PDS-H q7's `by { supp_nation, cust_nation, l_year }` + // emitted 8 groups instead of 4, the first chunk's four separated from + // the rest, because only the first chunk paid a reallocation. It needed + // multi-chunk input to show at all (`IBEX_CHUNK_ROWS`). + if (cat_interns_.size() < entries.size()) { + cat_interns_.resize(entries.size()); + } + PackedPlan plan; + plan.cols.reserve(entries.size()); + unsigned bytes = 0; + for (std::size_t k = 0; k < entries.size(); ++k) { + const auto& entry = *entries[k]; + PackCol col; + col.shift = bytes * 8; + const ColumnValue& column = *entry.column; + if (const auto* c_int = std::get_if>(&column)) { + col.kind = PackCol::Kind::Int64; + col.i64 = c_int->data(); + bytes += 8; + } else if (const auto* c_date = std::get_if>(&column)) { + col.kind = PackCol::Kind::Date; + col.date = c_date->data(); + bytes += 4; + } else if (const auto* c_ts = std::get_if>(&column)) { + col.kind = PackCol::Kind::Ts; + col.ts = c_ts->data(); + bytes += 8; + } else if (const auto* c_bool = std::get_if>(&column)) { + col.kind = PackCol::Kind::Bool; + col.boolean = c_bool; + bytes += 1; + } else if (const auto* c_cat = std::get_if>(&column)) { + col.remap = intern_categorical(k, *c_cat); + // An empty dictionary would leave the row loop indexing a remap + // that has no entry for any code. Declining here keeps the hot + // loop free of a per-row range check. + if (col.remap == nullptr) { + return std::nullopt; + } + col.kind = PackCol::Kind::Cat; + col.cat = c_cat; + bytes += 4; + } else { + return std::nullopt; + } + if (bytes > sizeof(Packed256)) { + return std::nullopt; + } + plan.cols.push_back(col); + } + plan.width = bytes; + return plan; + } + + /// Resolve chunk-local codes of key column `k` to operator-global ids, + /// returning the remap, or nullptr when the dictionary is empty. Runs once + /// per chunk per categorical key column. + auto intern_categorical(std::size_t k, const Column& cat) -> const std::uint32_t* { + // Never grows `cat_interns_` — `build_packed_key` sized it before any + // caller took a pointer into it, and growing here would dangle those. + auto& state = cat_interns_[k]; + const auto& dict = cat.dictionary(); + const std::size_t dict_size = dict.size(); + if (dict_size == 0) { + return nullptr; + } + state.remap.resize(dict_size); + for (std::size_t code = 0; code < dict_size; ++code) { + const std::string_view value = dict[code]; + if (const auto it = state.ids.find(value); it != state.ids.end()) { + state.remap[code] = it->second; + continue; + } + const auto id = static_cast(state.ids.size()); + // The view must outlive the chunk's dictionary, so the map keys are + // views into this deque rather than into the column. + state.arena.emplace_back(value); + state.ids.emplace(std::string_view{state.arena.back()}, id); + state.remap[code] = id; + } + return state.remap.data(); + } + + /// Pack one row's key. Cheap enough — a handful of array reads and shifts, + /// no hashing and no branch on width — that callers which need the key + /// twice recompute it rather than materialize a buffer. + template + [[nodiscard]] static auto pack_row(const std::vector& cols, std::size_t row) + -> Packed { + Packed key{}; + for (const auto& col : cols) { + std::uint64_t cell = 0; + switch (col.kind) { + case PackCol::Kind::Int64: + cell = static_cast(col.i64[row]); + break; + case PackCol::Kind::Date: + cell = static_cast(col.date[row].days); + break; + case PackCol::Kind::Ts: + cell = static_cast(col.ts[row].nanos); + break; + case PackCol::Kind::Bool: + cell = (*col.boolean)[row] ? 1U : 0U; + break; + case PackCol::Kind::Cat: + cell = col.remap[static_cast(col.cat->code_at(row))]; + break; + } + if constexpr (std::is_same_v) { + key |= cell << col.shift; + } else { + splice(key, cell, col.shift); + } + } + return key; + } + + /// Materialize this chunk's packed key for every row in `[begin, end)`. + template + static void build_keys(const std::vector& cols, std::size_t begin, std::size_t end, + Packed* out) { + for (std::size_t row = begin; row < end; ++row) { + out[row] = pack_row(cols, row); + } + } + + /// Interning state, indexed by key column position (see `CatIntern`). + std::vector cat_interns_; +}; + +} // namespace ibex::runtime diff --git a/src/runtime/physical_executor.cpp b/src/runtime/physical_executor.cpp new file mode 100644 index 00000000..47b94eac --- /dev/null +++ b/src/runtime/physical_executor.cpp @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +#include "interpreter_internal.hpp" +#include "physical_executor_internal.hpp" + +namespace ibex::runtime { + +auto build_migrated_physical_operator(const physical::Plan& plan, const ir::Node& node, + const TableRegistry& registry, const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec, + ModelResult* model_out) + -> std::expected { + if (!plan.migrated) { + return std::unexpected("physical executor: plan does not migrate its root"); + } + if (plan.root != &node) { + return std::unexpected("physical executor: plan root does not match execution root"); + } + + using namespace physical_executor_detail; + if (node.kind() == ir::NodeKind::Head) { + physical::note_map_pipeline_executed(); + return build_physical_head(node, registry, scalars, externs, exec, model_out); + } + if (node.kind() == ir::NodeKind::Tail) { + physical::note_map_pipeline_executed(); + return build_physical_tail(node, registry, scalars, externs, exec, model_out); + } + if (node.kind() == ir::NodeKind::TopK) { + physical::note_map_pipeline_executed(); + return build_physical_topk(node, registry, scalars, externs, exec, model_out); + } + if (node.kind() == ir::NodeKind::FilterHead || node.kind() == ir::NodeKind::FilterTail) { + physical::note_map_pipeline_executed(); + return build_physical_filter_head_tail(node, registry, scalars, externs, exec, model_out); + } + if (node.kind() == ir::NodeKind::Distinct) { + physical::note_map_pipeline_executed(); + return build_physical_distinct(plan, node, registry, scalars, externs, exec, model_out); + } + if (node.kind() == ir::NodeKind::Order) { + physical::note_map_pipeline_executed(); + return build_physical_order(node, registry, scalars, externs, exec, model_out); + } + if (plan.aggregate.describes) { + physical::note_map_pipeline_executed(); + return build_physical_aggregate(plan, node, registry, scalars, externs, exec, model_out); + } + if (plan.join.describes) { + physical::note_map_pipeline_executed(); + return build_physical_join(plan, node, registry, scalars, externs, exec, model_out); + } + if (plan.mode != physical::PipelineMode::MorselParallel || !exec.can_fan_out()) { + physical::note_map_pipeline_executed(); + } + return build_physical_map_step(plan, 0, registry, scalars, externs, exec, model_out); +} + +auto build_operator_from_physical_plan(const physical::Plan& plan, const ir::Node& node, + const TableRegistry& registry, const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec, + ModelResult* model_out) + -> std::expected { + return build_migrated_physical_operator(plan, node, registry, scalars, externs, exec, + model_out); +} + +} // namespace ibex::runtime diff --git a/src/runtime/physical_executor_internal.hpp b/src/runtime/physical_executor_internal.hpp new file mode 100644 index 00000000..b5f790ba --- /dev/null +++ b/src/runtime/physical_executor_internal.hpp @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +#pragma once + +#include +#include + +#include +#include +#include + +#include "physical_plan.hpp" + +namespace ibex::runtime { + +/// Validate and execute a plan whose root is owned by the physical executor. +[[nodiscard]] auto build_migrated_physical_operator( + const physical::Plan& plan, const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, const ExecutionContext& exec, + ModelResult* model_out) -> std::expected; + +/// Construction primitives supplied by the operator and pipeline families. +/// The physical executor owns dispatch; these functions keep concrete operator +/// types in the translation units that implement them. +namespace physical_executor_detail { + +[[nodiscard]] auto resolved_join_parallelism(const ExecutionContext& exec) + -> physical::JoinParallelism; + +[[nodiscard]] auto build_physical_map_step(const physical::Plan& plan, std::size_t index, + const TableRegistry& registry, + const ScalarRegistry* scalars, + const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_join(const physical::Plan& plan, const ir::Node& node, + const TableRegistry& registry, const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec, + ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_aggregate(const physical::Plan& plan, const ir::Node& node, + const TableRegistry& registry, + const ScalarRegistry* scalars, + const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_order(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec, + ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_head(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_tail(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_topk(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_filter_head_tail( + const ir::Node& node, const TableRegistry& registry, const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto build_physical_distinct(const physical::Plan& plan, const ir::Node& node, + const TableRegistry& registry, + const ScalarRegistry* scalars, + const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +} // namespace physical_executor_detail +} // namespace ibex::runtime diff --git a/src/runtime/physical_plan.cpp b/src/runtime/physical_plan.cpp index eb292a13..0d8dc29e 100644 --- a/src/runtime/physical_plan.cpp +++ b/src/runtime/physical_plan.cpp @@ -14,11 +14,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -68,30 +70,30 @@ constexpr std::size_t kJoinBuildRowFloor = 1U << 17U; constexpr std::size_t kJoinProbeRowFloor = 1U << 14U; constexpr std::size_t kJoinMaxWorkers = 64; -/// A hash aggregate's two fan-out floors, matching the private constants in -/// `chunked.cpp`. `kAggPartitionRowFloor` is the general radix path's +/// Hash-aggregate fan-out floors, matching the private constants in +/// `chunked.cpp`. `kAggPartitionRowFloor` is Discovery's general radix path's /// `kDefaultPartitionMinRows` -- the row count below which `try_discover_ /// partitioned` stays serial. `try_owned`'s lower `kPairOwnedMinRows` (65536) /// is not the phase floor: it is the operator-resolved "is the owned /// specialization worth it" gate, the same kind of runtime strategy choice the /// join operator makes for its build orientation, and it stays in the operator. /// `kAggFinalizeRowFloor` matches the `1U << 17U` group-count gate on -/// `finalize_owned`'s parallel co-ranking merge. Both phases share the -/// `min(budget, pool, 64)` worker cap that is open-coded at ~7 sites in -/// `ChunkedAggregateOperator` today. +/// `finalize_owned`'s parallel co-ranking merge. Discovery, Accumulation, and +/// FinalOrdering retain the existing 64-worker ceiling; Emission is bounded by +/// its output-column count and the shared compute budget. constexpr std::size_t kAggPartitionRowFloor = 1U << 18U; constexpr std::size_t kAggFinalizeRowFloor = 1U << 17U; constexpr std::size_t kAggMaxWorkers = 64; -/// A footer row estimate for `distinct`'s input, or `None` when the input is -/// not a bare registered scan. Deliberately conservative: a Filter or Join -/// under the Distinct makes the count unknowable at plan time, and the planner -/// never guesses — `None` means the operator decides on its first chunk, which -/// is exactly today's behaviour. -auto distinct_row_estimate(const ir::Node& distinct, const TableRegistry& registry) -> RowEstimate { - const ir::Node* cur = distinct.children().empty() ? nullptr : distinct.children().front().get(); +/// The exact cardinality of a registered table scan, or `None` when its input +/// cardinality is altered or otherwise unavailable. Project, Rename, and a +/// row-local Update preserve row count; a Filter or Join deliberately makes it +/// unknowable at plan time. +auto table_input_row_estimate(const ir::Node& root, const TableRegistry& registry) -> RowEstimate { + const ir::Node* cur = root.children().empty() ? nullptr : root.children().front().get(); while (cur != nullptr && - (cur->kind() == ir::NodeKind::Project || cur->kind() == ir::NodeKind::Rename)) { + (cur->kind() == ir::NodeKind::Project || cur->kind() == ir::NodeKind::Rename || + cur->kind() == ir::NodeKind::Update)) { cur = cur->children().empty() ? nullptr : cur->children().front().get(); } if (cur == nullptr || cur->kind() != ir::NodeKind::Scan) { @@ -102,7 +104,97 @@ auto distinct_row_estimate(const ir::Node& distinct, const TableRegistry& regist if (it == registry.end()) { return {}; } - return {.rows = it->second.rows(), .source = RowEstimate::Source::Footer}; + return {.rows = it->second.rows(), .source = RowEstimate::Source::TableExact}; +} + +auto runtime_column_type(const ColumnValue& column) -> ir::ColumnType { + return std::visit( + [](const ColumnT&) { + using ColumnType = std::remove_cvref_t; + if constexpr (std::same_as>) { + return ir::ColumnType::Int64; + } else if constexpr (std::same_as>) { + return ir::ColumnType::Float64; + } else if constexpr (std::same_as>) { + return ir::ColumnType::Bool; + } else if constexpr (std::same_as>) { + return ir::ColumnType::String; + } else if constexpr (std::same_as>) { + return ir::ColumnType::Categorical; + } else if constexpr (std::same_as>) { + return ir::ColumnType::Date; + } else { + static_assert(std::same_as>); + return ir::ColumnType::Timestamp; + } + }, + column); +} + +auto planning_source_schemas(const TableRegistry& registry, const ir::SourceSchemas& declared) + -> ir::SourceSchemas { + ir::SourceSchemas schemas = declared; + schemas.reserve(schemas.size() + registry.size()); + for (const auto& [name, table] : registry) { + std::vector fields; + fields.reserve(table.columns.size()); + for (const ColumnEntry& column : table.columns) { + fields.push_back({.name = column.name, .type = runtime_column_type(*column.column)}); + } + schemas.insert_or_assign(name, ir::SchemaInfo::known(std::move(fields), /*open=*/false)); + } + return schemas; +} + +auto known_join_column_mapping(const ir::JoinNode& join, const ir::SourceSchemas& schemas) + -> std::optional { + if (join.children().size() != 2) { + return std::nullopt; + } + const ir::SchemaInfo left = ir::infer_schema(*join.children()[0], schemas); + const ir::SchemaInfo right = ir::infer_schema(*join.children()[1], schemas); + if (!left.is_known() || left.is_open() || !right.is_known() || right.is_open()) { + return std::nullopt; + } + std::vector left_names; + left_names.reserve(left.fields().size()); + for (const ir::SchemaField& field : left.fields()) { + left_names.push_back(field.name); + } + std::vector right_names; + right_names.reserve(right.fields().size()); + for (const ir::SchemaField& field : right.fields()) { + right_names.push_back(field.name); + } + auto mapping = + ir::resolve_join_columns(join.kind(), join.keys(), left_names, right_names, join.suffix()); + if (!mapping.has_value()) { + return std::nullopt; + } + return std::move(*mapping); +} + +auto known_aggregate_column_mapping(const ir::AggregateNode& aggregate, + const ir::SourceSchemas& schemas) + -> std::optional { + if (aggregate.children().size() != 1) { + return std::nullopt; + } + const ir::SchemaInfo input = ir::infer_schema(*aggregate.children().front(), schemas); + if (!input.is_known() || input.is_open()) { + return std::nullopt; + } + std::vector input_names; + input_names.reserve(input.fields().size()); + for (const ir::SchemaField& field : input.fields()) { + input_names.push_back(field.name); + } + auto mapping = + resolve_aggregate_columns(aggregate.group_by(), aggregate.aggregations(), input_names); + if (!mapping.has_value()) { + return std::nullopt; + } + return std::move(*mapping); } auto join_strategy_name(JoinStrategy strategy) -> std::string_view { @@ -412,9 +504,10 @@ void resolve_pipeline_mode(Plan& plan) { } // namespace auto plan_physical(const ir::Node& root, const TableRegistry& registry, - const ExternRegistry* externs) -> Plan { + const ExternRegistry* externs, const ir::SourceSchemas& source_schemas) -> Plan { Plan plan; plan.root = &root; + const ir::SourceSchemas schemas = planning_source_schemas(registry, source_schemas); plan_stats().plans_built.fetch_add(1, std::memory_order_relaxed); // Describe a join even though the plan does not execute one yet. The plan @@ -446,7 +539,7 @@ auto plan_physical(const ir::Node& root, const TableRegistry& registry, plan.source_node = &root; plan.breaker_phases.push_back( {.name = "dedup", - .parallelism = distinct_dedup_parallelism(distinct_row_estimate(root, registry))}); + .parallelism = distinct_dedup_parallelism(table_input_row_estimate(root, registry))}); return plan; } if (root.kind() == ir::NodeKind::Order) { @@ -461,36 +554,44 @@ auto plan_physical(const ir::Node& root, const TableRegistry& registry, return plan; } if (root.kind() == ir::NodeKind::Aggregate) { - // Described, not executed: `migrated` stays false and the per-kind - // switch still builds every aggregate. The description is proven equal - // to the builder's branches first, exactly as the join's was. - plan.aggregate = plan_aggregate(ir::node_cast(root)); + const auto& aggregate = ir::node_cast(root); + plan.aggregate = plan_aggregate(aggregate); // Streaming and fused aggregates are executed by the plan now. // `MaterializeAll` is not: it still falls back and still counts, which // is what keeps the backlog measuring the port rather than the label. if (plan.aggregate.strategy != AggregateStrategy::MaterializeAll) { plan.migrated = true; plan.source_node = &root; - // Describe the hash aggregate's fan-out points. Only the streamed - // path reaches `ChunkedAggregateOperator` (via - // `ChunkedSortedAggregateOperator`'s hash fallback); the fused - // left-join count runs whole-table and has no fan-out to describe. - // Descriptive only (slice 1): `build_physical_aggregate` does not - // read these, the constants still live in `chunked.cpp`, and the - // operator aborts via `check_agg_plan` if its own decision ever - // disagrees. The authority slices move the decision the way - // distinct's and the join's did. + // Plan the adaptive streamable aggregate's structural nodes and + // attach the fan-out policy each node owns. The fused left-join + // count is whole-table and has no hash fallback to describe. if (plan.aggregate.strategy == AggregateStrategy::StreamingSorted) { - plan.breaker_phases.push_back( - {.name = "partition", .parallelism = aggregate_partition_parallelism()}); - plan.breaker_phases.push_back( - {.name = "finalize", .parallelism = aggregate_finalize_parallelism()}); + plan.aggregate.columns = known_aggregate_column_mapping(aggregate, schemas); + const RowEstimate input_estimate = table_input_row_estimate(root, registry); + plan.hash_aggregate = HashAggregateNodes{ + .discovery = {.source = aggregate.children().front().get(), + .input = AggregateDataKind::InputChunks, + .output = AggregateDataKind::DiscoveredGroups, + .parallelism = aggregate_discovery_parallelism(input_estimate)}, + .accumulation = {.input = AggregateDataKind::DiscoveredGroups, + .output = AggregateDataKind::AccumulatedGroups, + .parallelism = + aggregate_accumulation_parallelism(input_estimate)}, + .final_ordering = {.input = AggregateDataKind::AccumulatedGroups, + .output = AggregateDataKind::OrderedGroups, + .parallelism = + aggregate_final_ordering_parallelism(input_estimate)}, + .emission = {.input = AggregateDataKind::OrderedGroups, + .output = AggregateDataKind::OutputChunks, + .parallelism = aggregate_emission_parallelism(input_estimate)}, + }; } return plan; } } if (root.kind() == ir::NodeKind::Join) { - plan.join = plan_join(ir::node_cast(root)); + const auto& join = ir::node_cast(root); + plan.join = plan_join(join); // A streaming join is executed by the plan now: `build_physical_join` // builds it, not the per-kind switch. A materializing one is still a // fallback and says so, which is why the backlog drops by the streaming @@ -498,14 +599,24 @@ auto plan_physical(const ir::Node& root, const TableRegistry& registry, if (plan.join.strategy == JoinStrategy::StreamingProbe) { plan.migrated = true; plan.source_node = &root; - // Describe the two fan-out points the join operator already has. - // Descriptive only (slice 3): `build_physical_join` does not read - // these; the constants still live in `chunked.cpp`. A follow-up - // slice moves the authority, the way distinct's dedup phase did. - plan.breaker_phases.push_back( - {.name = "hash-build", .parallelism = join_hash_build_parallelism()}); - plan.breaker_phases.push_back( - {.name = "probe", .parallelism = join_probe_parallelism()}); + // An inner join is two explicit physical nodes joined by a typed + // build-output edge. Both retain the textual inputs because + // orientation is resolved only after the build has measured them. + // Semi/anti still uses its separate operator and is not described + // by the inner join's runtime-oriented output type. + if (plan.join.branch != JoinBranch::SemiAnti) { + plan.streaming_join = StreamingJoinNodes{ + .build = {.left_input = plan.join.left_input, + .right_input = plan.join.right_input, + .output = JoinDataKind::RuntimeOrientedBuildOutput, + .parallelism = join_hash_build_parallelism()}, + .probe = {.build_input = JoinDataKind::RuntimeOrientedBuildOutput, + .left_input = plan.join.left_input, + .right_input = plan.join.right_input, + .parallelism = join_probe_parallelism()}, + .columns = known_join_column_mapping(join, schemas), + }; + } return plan; } } @@ -649,6 +760,13 @@ auto explain_physical(const Plan& plan) -> std::string { std::string out; if (!plan.migrated) { out += "MaterializedCall("; + // Name the logical subtree the fallback retains (physical_plan.hpp): a + // join it fully understands and a node it knows nothing about must not + // print the same opaque line. `plan.root` is always set by `plan_physical`. + if (plan.root != nullptr) { + out += node_kind_name_impl(plan.root->kind()); + out += ": "; + } out += fallback_reason_name(plan.reason); out += ")\n"; // A described-but-not-executed node still explains itself. Without @@ -670,19 +788,53 @@ auto explain_physical(const Plan& plan) -> std::string { if (plan.steps.empty() && plan.root != nullptr) { const std::string_view kind = node_kind_name_impl(plan.root->kind()); if (plan.join.describes) { - // The strategy line, then one line per fan-out phase (a streaming - // join carries hash-build + probe; see `plan_physical`). + // The strategy line, explicit build → probe edge, then each node's + // fan-out policy. `breaker_phases` remains for untyped breakers; + // a streaming join no longer hides its dataflow in two labels. out += "Breaker(Join)\n " + explain_join(plan.join); - append_phase_lines(out, plan.breaker_phases); + if (plan.streaming_join.has_value()) { + out += "\n edge: HashBuild.RuntimeOrientedBuildOutput -> HashProbe.build_input"; + if (plan.streaming_join->columns.has_value()) { + out += "\n columns: resolved"; + for (const ir::JoinKeyColumns& key : plan.streaming_join->columns->keys) { + out += " left[" + std::to_string(key.left_index) + "]=right[" + + std::to_string(key.right_index) + "]"; + } + out += + " output=" + std::to_string(plan.streaming_join->columns->output.size()); + } else { + out += "\n columns: deferred (bind once from concrete inputs)"; + } + const std::vector nodes{ + {.name = "HashBuild", .parallelism = plan.streaming_join->build.parallelism}, + {.name = "HashProbe", .parallelism = plan.streaming_join->probe.parallelism}, + }; + append_phase_lines(out, nodes); + } out += '\n'; return out; } if (plan.aggregate.describes) { - // The strategy line, then one line per fan-out phase (the streamed - // hash path carries partition + finalize; see `plan_physical`). A - // fused left-join count has no phases and prints just the strategy. + // The adaptive path shows its structural hash-fallback chain and + // each node's own fan-out policy. A fused left-join count has + // neither and prints just the strategy. out += "Breaker(Aggregate)\n " + explain_aggregate(plan.aggregate); - append_phase_lines(out, plan.breaker_phases); + if (plan.hash_aggregate.has_value()) { + out += "\n hash-fallback: Discovery -> Accumulation -> FinalOrdering -> Emission"; + out += + "\n edge: InputChunks -> DiscoveredGroups -> AccumulatedGroups -> " + "OrderedGroups -> OutputChunks"; + const std::vector nodes{ + {.name = "Discovery", + .parallelism = plan.hash_aggregate->discovery.parallelism}, + {.name = "Accumulation", + .parallelism = plan.hash_aggregate->accumulation.parallelism}, + {.name = "FinalOrdering", + .parallelism = plan.hash_aggregate->final_ordering.parallelism}, + {.name = "Emission", .parallelism = plan.hash_aggregate->emission.parallelism}, + }; + append_phase_lines(out, nodes); + } out += '\n'; return out; } @@ -768,6 +920,41 @@ auto explain_physical(const Plan& plan) -> std::string { return out; } +auto resolve_aggregate_columns(std::span group_by, + std::span aggregations, + std::span input_names) + -> std::expected { + AggregateColumnMapping mapping; + mapping.input_names.reserve(input_names.size()); + for (const std::string_view name : input_names) { + mapping.input_names.emplace_back(name); + } + mapping.group_by.reserve(group_by.size()); + for (const ir::ColumnRef& key : group_by) { + const auto found = std::ranges::find(input_names, key.name); + if (found == input_names.end()) { + return std::unexpected("group-by column not found: " + key.name); + } + mapping.group_by.push_back( + static_cast(std::distance(input_names.begin(), found))); + } + + mapping.aggregate_inputs.reserve(aggregations.size()); + for (const ir::AggSpec& aggregation : aggregations) { + if (aggregation.func == ir::AggFunc::Count) { + mapping.aggregate_inputs.emplace_back(std::nullopt); + continue; + } + const auto found = std::ranges::find(input_names, aggregation.column.name); + if (found == input_names.end()) { + return std::unexpected("aggregate column not found: " + aggregation.column.name); + } + mapping.aggregate_inputs.emplace_back( + static_cast(std::distance(input_names.begin(), found))); + } + return mapping; +} + auto plan_aggregate(const ir::AggregateNode& agg) -> AggregatePlan { AggregatePlan out; out.describes = true; @@ -815,6 +1002,10 @@ auto partition_strategy_name(PartitionStrategy strategy) -> std::string_view { return "head-table (partition by key hash)"; case PartitionStrategy::Range: return "range (contiguous probe-row slices)"; + case PartitionStrategy::Morsel: + return "morsel (deterministic row ranges)"; + case PartitionStrategy::Column: + return "column (independent output columns)"; } return "?"; } @@ -845,20 +1036,33 @@ auto join_probe_parallelism() -> BreakerParallelism { .strategy = PartitionStrategy::Range}; } -auto aggregate_partition_parallelism() -> BreakerParallelism { +auto aggregate_discovery_parallelism(RowEstimate estimate) -> BreakerParallelism { // RadixHash is the general strategy; `try_owned` is a specialization of it // (partition-owned key maps instead of whole scattered partitions). The // operator resolves which at run time from the key type and cardinality, // the way the join resolves its build orientation. return {.row_floor = kAggPartitionRowFloor, .breaker_max_workers = kAggMaxWorkers, - .strategy = PartitionStrategy::RadixHash}; + .strategy = PartitionStrategy::RadixHash, + .estimate = estimate}; } -auto aggregate_finalize_parallelism() -> BreakerParallelism { +auto aggregate_accumulation_parallelism(RowEstimate estimate) -> BreakerParallelism { return {.row_floor = kAggFinalizeRowFloor, .breaker_max_workers = kAggMaxWorkers, - .strategy = PartitionStrategy::Owned}; + .strategy = PartitionStrategy::Morsel, + .estimate = estimate}; +} + +auto aggregate_final_ordering_parallelism(RowEstimate estimate) -> BreakerParallelism { + return {.row_floor = kAggFinalizeRowFloor, + .breaker_max_workers = kAggMaxWorkers, + .strategy = PartitionStrategy::Owned, + .estimate = estimate}; +} + +auto aggregate_emission_parallelism(RowEstimate estimate) -> BreakerParallelism { + return {.strategy = PartitionStrategy::Column, .estimate = estimate}; } void resolve_breaker_parallelism(BreakerParallelism& bp, const ExecutionContext& exec, @@ -919,9 +1123,26 @@ void append_phase_lines(std::string& out, const std::vector& phase } out += " partitions=derived "; out += partition_strategy_name(bp.strategy); - out += bp.estimate.confident() - ? "\n estimate " + std::to_string(bp.estimate.rows) + " rows (footer)" - : "\n no row estimate -> decided on first chunk"; + if (!bp.estimate.confident()) { + out += "\n no row estimate -> decided on first chunk"; + continue; + } + out += "\n input estimate " + std::to_string(bp.estimate.rows) + " rows ("; + switch (bp.estimate.source) { + case RowEstimate::Source::Footer: + out += "footer"; + break; + case RowEstimate::Source::TableExact: + out += "table"; + break; + case RowEstimate::Source::ChildExact: + out += "child"; + break; + case RowEstimate::Source::None: + out += "?"; + break; + } + out += ")"; } } } // namespace @@ -935,6 +1156,45 @@ auto explain_breaker(std::string_view kind, const std::vector& pha return out; } +auto validate_streaming_join_edge(const StreamingJoinNodes& nodes) -> std::optional { + if (nodes.build.left_input == nullptr || nodes.build.right_input == nullptr || + nodes.probe.left_input == nullptr || nodes.probe.right_input == nullptr) { + return "physical join: HashBuild/HashProbe edge has a missing candidate input"; + } + if (nodes.build.left_input != nodes.probe.left_input || + nodes.build.right_input != nodes.probe.right_input) { + return "physical join: HashBuild and HashProbe candidate inputs disagree"; + } + if (nodes.build.output != JoinDataKind::RuntimeOrientedBuildOutput || + nodes.probe.build_input != nodes.build.output) { + return "physical join: HashProbe does not consume HashBuild's runtime-oriented output"; + } + return std::nullopt; +} + +auto validate_hash_aggregate_edges(const HashAggregateNodes& nodes) -> std::optional { + if (nodes.discovery.source == nullptr) { + return "physical aggregate: Discovery has no input"; + } + if (nodes.discovery.input != AggregateDataKind::InputChunks || + nodes.discovery.output != AggregateDataKind::DiscoveredGroups || + nodes.accumulation.input != nodes.discovery.output) { + return "physical aggregate: Discovery -> Accumulation edge is invalid"; + } + if (nodes.accumulation.output != AggregateDataKind::AccumulatedGroups || + nodes.final_ordering.input != nodes.accumulation.output) { + return "physical aggregate: Accumulation -> FinalOrdering edge is invalid"; + } + if (nodes.final_ordering.output != AggregateDataKind::OrderedGroups || + nodes.emission.input != nodes.final_ordering.output) { + return "physical aggregate: FinalOrdering -> Emission edge is invalid"; + } + if (nodes.emission.output != AggregateDataKind::OutputChunks) { + return "physical aggregate: Emission output is invalid"; + } + return std::nullopt; +} + auto plan_join(const ir::JoinNode& join) -> JoinPlan { JoinPlan out; out.describes = true; diff --git a/src/runtime/physical_plan.hpp b/src/runtime/physical_plan.hpp index 352db140..22511143 100644 --- a/src/runtime/physical_plan.hpp +++ b/src/runtime/physical_plan.hpp @@ -3,33 +3,37 @@ #pragma once +#include #include +#include #include #include #include #include #include +#include +#include +#include #include #include #include namespace ibex::runtime::physical { -/// Phase 1 of plans/kernel-pipeline-execution-plan.md: an explicit, data-only -/// physical plan computed *beside* `build_operator`, before any of its -/// per-kind branches run. The plan is the decision record — which pipelines -/// exist, and why anything else materialized — not a second executor. Phase 2 -/// replaces per-step execution with kernels; until then a migrated pipeline -/// constructs exactly the operators the per-kind switch would have. +/// An explicit, data-only physical plan computed before execution construction. +/// The plan is the decision record — which pipelines and breaker phases exist, +/// and why anything else materialized. The physical-plan execution adapter +/// consumes migrated plans; residual shapes use an explicit materialized-call +/// fallback. /// -/// Scope (deliberately tiny, per the plan's "first vocabulary"): -/// a `MapPipeline` is a non-empty top-down chain of row-local map nodes +/// A `MapPipeline` is a non-empty top-down chain of row-local map nodes /// (Filter, Project, Rename, and row-local Update) over a source. A source is /// a Scan, a chunked extern call, or the materialized output of a subtree that -/// keeps the existing executor — a pipeline breaker feeding this pipeline. -/// Everything else is a `MaterializedCall` placeholder naming the logical -/// subtree that keeps the existing executor. +/// keeps the residual executor — a pipeline breaker feeding this pipeline. +/// Migrated breakers add explicit join and aggregate phase nodes plus the +/// single-operator breaker descriptions; everything else is a +/// `MaterializedCall` naming the logical subtree retained by the fallback. enum class SourceKind : std::uint8_t { TableScan, ///< Scan of a table in the registry LazyScan, ///< Scan resolved lazily (deferred/reader-backed) @@ -152,11 +156,9 @@ enum class JoinBranch : std::uint8_t { /// What the plan knows about a `Join` node. /// -/// Describes only. Execution still goes through `build_operator`'s per-kind -/// switch, exactly as before -- this is the same order the island removal took -/// (describe, prove the description equals what the builder does, then move -/// execution), because a description that is wrong is much cheaper to find than -/// an executor that is wrong. +/// Streaming execution consumes this description through the physical-plan +/// adapter. Materializing and semi/anti shapes retain their explicit fallback +/// construction paths. struct JoinPlan { /// False unless the planned node is a `Join`. bool describes = false; @@ -197,14 +199,32 @@ enum class AggregateStrategy : std::uint8_t { /// Checked first, because it consumes the join rather than reading its /// output. FusedLeftJoinCount, - /// Streamed group-at-a-time; falls back to hashing internally when the - /// child's chunks do not arrive sorted on the group keys. + /// Adaptive streamable aggregate: group-at-a-time when the child's chunks + /// arrive sorted on the group keys, hash aggregation otherwise. The enum + /// name predates the hash fallback becoming the common generated-data path. StreamingSorted, /// Median and Quantile need every value at once, Ewma is row-order /// coupled: the whole input is materialized. MaterializeAll, }; +/// One aggregate input resolved against the ordered physical columns of its +/// child. Logical names stop at this boundary; execution phases share these +/// positions. +struct AggregateColumnMapping { + /// Ordered child layout against which the positions below were resolved. + /// A lazy physical source may legitimately project predicate-only columns + /// away after logical schema inference; the executor detects that boundary + /// change once and re-resolves the positions against the concrete chunk. + std::vector input_names; + std::vector group_by; + /// Count consumes rows rather than an input column, so its entry is + /// nullopt. Every other aggregate carries its input column position. + std::vector> aggregate_inputs; + + auto operator==(const AggregateColumnMapping&) const -> bool = default; +}; + /// What the plan knows about an `Aggregate` node. /// /// Every field is RELAYED from the predicates the builder itself calls -- @@ -222,8 +242,20 @@ struct AggregatePlan { /// The column the fused count/sum reads, resolved back through any updates /// between the aggregate and the join. Empty unless fused. std::string counted_column; + /// Positional bindings for the streamed/hash aggregate input. Closed known + /// schemas fill this during planning; lazy/unknown inputs bind it once at + /// the first concrete chunk. + std::optional columns; }; +/// Resolve an aggregate's logical column references against one ordered input +/// schema. The same function serves physical planning and the one-time runtime +/// bind for inputs whose schema is not statically known. +[[nodiscard]] auto resolve_aggregate_columns(std::span group_by, + std::span aggregations, + std::span input_names) + -> std::expected; + /// Classify an aggregate. Pure, and decided entirely by relayed predicates. [[nodiscard]] auto plan_aggregate(const ir::AggregateNode& agg) -> AggregatePlan; @@ -253,6 +285,8 @@ enum class PartitionStrategy : std::uint8_t { ///< (`PartitionedHeads`), one worker filling each — no merge afterwards Range, ///< a worker per contiguous slice of probe rows, output concatenated in ///< range order (`ChunkedInnerJoinOperator::probe_ranges_parallel`) + Morsel, ///< deterministic row-derived morsels with private reduction state + Column, ///< one independent output column per task }; /// A row-count estimate available at plan time, and where it came from. The @@ -260,7 +294,7 @@ enum class PartitionStrategy : std::uint8_t { /// means the operator will decide on its first chunk (today's behaviour). struct RowEstimate { std::size_t rows = 0; - enum class Source : std::uint8_t { None, Footer, ChildExact } source = Source::None; + enum class Source : std::uint8_t { None, Footer, TableExact, ChildExact } source = Source::None; [[nodiscard]] auto confident() const noexcept -> bool { return source != Source::None; } }; @@ -288,9 +322,9 @@ struct BreakerParallelism { std::size_t worker_cap = 0; }; -/// A breaker is one or more named phases, each with a fan-out point. Distinct / -/// Order / TopK have one; a decomposed Join has two (hash-build, probe); a -/// decomposed Aggregate has three (discovery, accumulate, finalize). +/// An untyped breaker is one or more named phases, each with a fan-out point. +/// Distinct and Order have one. Join and Aggregate attach policy directly to +/// their typed structural nodes instead. struct BreakerPhase { std::string_view name; BreakerParallelism parallelism; @@ -332,30 +366,122 @@ struct JoinParallelism { BreakerParallelism probe; }; -/// A hash aggregate's fan-out points. The operator has two today: -/// -/// - `partition` — the histogram → prefix-sum → scatter → per-partition -/// accumulate region shared by `try_discover_partitioned` (radix-hash) and -/// `try_owned` (partition-owned key maps). Discovery and accumulation are one -/// `pool.submit` here, not two; the spec's separate "discovery" / "accumulate" -/// phases appear only once that region is actually decomposed (Phase 5). -/// `row_floor` is the *lower* admission floor (65536, `try_owned`'s -/// `kPairOwnedMinRows`); the radix path's stricter 262144 -/// (`kDefaultPartitionMinRows`) is a strategy-internal detail slice 2 folds in. -/// - `finalize` — the K-way first-occurrence merge (`finalize_owned`'s -/// co-ranking merge, the ordered-run finalize, the non-owned first-row seed -/// pass). `row_floor` 131072 (`1U << 17U`). -[[nodiscard]] auto aggregate_partition_parallelism() -> BreakerParallelism; -[[nodiscard]] auto aggregate_finalize_parallelism() -> BreakerParallelism; +/// The typed edge between a streaming join's two physical nodes. The build +/// resolves which candidate input is indexed and returns that orientation with +/// the immutable hash index; the probe consumes both as one value. +enum class JoinDataKind : std::uint8_t { + None, + RuntimeOrientedBuildOutput, +}; + +/// Explicit physical hash-build node. Both logical inputs are candidates: the +/// smaller side is selected only after their actual row counts are known. +struct HashBuildNode { + const ir::Node* left_input = nullptr; + const ir::Node* right_input = nullptr; + JoinDataKind output = JoinDataKind::None; + BreakerParallelism parallelism; +}; + +/// Explicit physical hash-probe node. `build_input` names the build node's +/// output rather than smuggling the dependency through one monolithic join +/// operator. The runtime orientation selects which candidate input is probed. +struct HashProbeNode { + JoinDataKind build_input = JoinDataKind::None; + const ir::Node* left_input = nullptr; + const ir::Node* right_input = nullptr; + BreakerParallelism parallelism; +}; + +/// The two nodes and their barrier edge. Kept together so a plan cannot carry +/// a build without its consumer (or reverse their policies by accident). +struct StreamingJoinNodes { + HashBuildNode build; + HashProbeNode probe; + /// Resolved during planning when both child schemas are closed and known; + /// otherwise bound once from the first concrete inputs at execution. + std::optional columns; +}; + +/// Validate the typed HashBuild -> HashProbe edge. The executor calls the same +/// predicate mutation tests use, so a malformed plan cannot silently fall back +/// to the coordinator's implicit handoff. +[[nodiscard]] auto validate_streaming_join_edge(const StreamingJoinNodes& nodes) + -> std::optional; + +/// Capability policies for the four structural hash-aggregate nodes. Runtime +/// data still selects specialized kernels and deterministic partition counts; +/// these values own fan-out permission and worker ceilings. +[[nodiscard]] auto aggregate_discovery_parallelism(RowEstimate estimate = {}) -> BreakerParallelism; +[[nodiscard]] auto aggregate_accumulation_parallelism(RowEstimate estimate = {}) + -> BreakerParallelism; +[[nodiscard]] auto aggregate_final_ordering_parallelism(RowEstimate estimate = {}) + -> BreakerParallelism; +[[nodiscard]] auto aggregate_emission_parallelism(RowEstimate estimate = {}) -> BreakerParallelism; /// Both fan-out phases of a hash aggregate, resolved together — what /// `build_physical_aggregate` hands the operator. Bundled like `JoinParallelism` /// so the two cannot be passed in the wrong order. struct AggregateParallelism { - BreakerParallelism partition; - BreakerParallelism finalize; + BreakerParallelism discovery; + BreakerParallelism accumulation; + BreakerParallelism final_ordering; + BreakerParallelism emission; }; +/// Values crossing the structural phases of the hash-aggregate fallback. +/// These are ownership types, not table schemas: each value is internal state +/// that only the following phase may consume. +enum class AggregateDataKind : std::uint8_t { + None, + InputChunks, + DiscoveredGroups, + AccumulatedGroups, + OrderedGroups, + OutputChunks, +}; + +struct AggregateDiscoveryNode { + const ir::Node* source = nullptr; + AggregateDataKind input = AggregateDataKind::None; + AggregateDataKind output = AggregateDataKind::None; + BreakerParallelism parallelism; +}; + +struct AggregateAccumulationNode { + AggregateDataKind input = AggregateDataKind::None; + AggregateDataKind output = AggregateDataKind::None; + BreakerParallelism parallelism; +}; + +struct AggregateFinalOrderingNode { + AggregateDataKind input = AggregateDataKind::None; + AggregateDataKind output = AggregateDataKind::None; + BreakerParallelism parallelism; +}; + +struct AggregateEmissionNode { + AggregateDataKind input = AggregateDataKind::None; + AggregateDataKind output = AggregateDataKind::None; + BreakerParallelism parallelism; +}; + +/// The hash fallback's structural chain. The current optimized kernels may +/// fuse work within an edge, but the plan and executor agree on these four +/// ownership boundaries. Each node carries the policy for fan-out performed +/// within that boundary. +struct HashAggregateNodes { + AggregateDiscoveryNode discovery; + AggregateAccumulationNode accumulation; + AggregateFinalOrderingNode final_ordering; + AggregateEmissionNode emission; +}; + +/// Validate the typed Discovery -> Accumulation -> FinalOrdering -> Emission +/// chain. Execution calls the same predicate used by mutation tests. +[[nodiscard]] auto validate_hash_aggregate_edges(const HashAggregateNodes& nodes) + -> std::optional; + /// Fill `bp`'s resolved half. `pool_size` is `process_worker_pool().size()`, or /// 0 when the caller declined to construct the pool for a serial query. The one /// implementation of the worker-cap clamp that used to be open-coded per @@ -405,25 +531,33 @@ struct Plan { } std::vector source_signature; const ir::Node* root = nullptr; - /// Set when `root` is a `Join`. The plan does not execute it yet, so this - /// coexists with `migrated == false`: the plan describes more of the query - /// than it runs, which is what makes the backlog shrinkable one kind at a - /// time instead of in one jump. + /// Set when `root` is a `Join`. Streaming joins are migrated and consumed + /// by the physical-plan adapter; unsupported materializing shapes remain + /// described but use the explicit fallback. JoinPlan join; + /// Present exactly for a streaming inner join. These are physical nodes, + /// not generic labels in `breaker_phases`: their typed edge makes the + /// build → probe dependency and its runtime-resolved orientation + /// inspectable. Semi/anti retains its separate streaming operator. + std::optional streaming_join; /// Set when `root` is an `Aggregate`. AggregatePlan aggregate; + /// Present for the adaptive streamable aggregate's hash fallback. These + /// are structural phase nodes and each carries its own fan-out policy. + std::optional hash_aggregate; /// Set when `root` is a breaker whose parallelism the plan describes. /// Empty otherwise. One entry per fan-out phase (see `BreakerPhase`). std::vector breaker_phases; }; -/// Lower `root` into a Phase 1 plan. Read-only over the IR, the registry, and +/// Lower `root` into a physical plan. Read-only over the IR, the registry, and /// the context; safe to call from tests and `explain` tooling with no intent -/// to execute. A step is admitted only when the per-kind switch would build -/// it as a map too (`is_map_step` mirrors that routing exactly), so the plan -/// can never claim a shape the executor would construct differently. +/// to execute. A map step is admitted only when the residual construction path +/// agrees (`is_map_step` mirrors that routing exactly), so the plan cannot +/// claim a shape the executor would construct differently. [[nodiscard]] auto plan_physical(const ir::Node& root, const TableRegistry& registry, - const ExternRegistry* externs) -> Plan; + const ExternRegistry* externs, + const ir::SourceSchemas& source_schemas = {}) -> Plan; /// Deterministic multi-line rendering for tests and debugging. A plan that /// cannot explain why it materialized is not an acceptable plan, so every @@ -448,8 +582,8 @@ struct Plan { [[nodiscard]] auto physical_materialized_calls() -> std::uint64_t; /// Record that a pipeline is executing (called once per plan that migrates, -/// whether the executor is the Phase 1 serial composer or the parallel -/// pipeline, which is the same pipeline's parallel mode). +/// whether the adapter selects the serial composer or the same pipeline's +/// parallel mode). void note_map_pipeline_executed(); /// Record a fallback: one logical node the physical plan does not describe, so diff --git a/src/runtime/pipeline_executor.cpp b/src/runtime/pipeline_executor.cpp new file mode 100644 index 00000000..d28a00bf --- /dev/null +++ b/src/runtime/pipeline_executor.cpp @@ -0,0 +1,2406 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "physical_plan.hpp" + +#if defined(__AVX2__) || defined(__BMI2__) +#include +#endif + +#include "chunk_conversion_internal.hpp" +#include "execution_profile_internal.hpp" +#include "interpreter_internal.hpp" +#include "join_chunked_internal.hpp" +#include "kernel_filter.hpp" +#include "kernel_types.hpp" +#include "physical_executor_internal.hpp" +#include "pipeline_executor_internal.hpp" +#include "runtime_internal.hpp" + +namespace ibex::runtime::pipeline_executor_detail { + +struct ChunkIdentity { + std::uint64_t sequence = 0; + std::size_t row_offset = 0; +}; + +[[nodiscard]] auto table_to_chunk(Table table, ChunkIdentity identity) -> Chunk { + auto chunk = runtime::table_to_chunk(std::move(table)); + chunk.sequence = identity.sequence; + chunk.row_offset = identity.row_offset; + return chunk; +} + +auto materialize_row_local(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected { + auto op = build_operator(node, registry, scalars, externs, exec, model_out); + if (!op.has_value()) { + return std::unexpected(std::move(op.error())); + } + return materialize_operator(std::move(op.value())); +} + +} // namespace ibex::runtime::pipeline_executor_detail + +namespace ibex::runtime { + +namespace pipeline_executor_detail { + +// Runtime-multithreading Phase 1, serial morsel slice. Owns the materialized +// input `Table` that the pipeline's `PartitionedTableSource` reads by pointer. +// `input_` is declared before `chain_` so the chain — which holds a raw +// pointer into `input_` — is destroyed first. +class OwningMorselPipelineOperator final : public Operator { + public: + OwningMorselPipelineOperator(std::unique_ptr
input, OperatorPtr chain) + : input_(std::move(input)), chain_(std::move(chain)) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + return chain_->next(); + } + + private: + std::unique_ptr
input_; + OperatorPtr chain_; +}; + +/// Serial-slice stand-in for the Phase 1 ordered merger. The current source +/// emits in sequence order, so validating the stream is enough to make a lost, +/// duplicated, or provenance-stripped morsel an immediate error. A later +/// concurrent merger replaces this with sequence-indexed buffering/release. +class SerialMorselOrderValidator final : public Operator { + public: + SerialMorselOrderValidator(OperatorPtr child, std::uint64_t expected_morsels, std::size_t grain) + : child_(std::move(child)), + expected_morsels_(expected_morsels), + grain_(grain == 0 ? 1 : grain) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + auto result = child_->next(); + if (!result.has_value()) { + return std::unexpected(std::move(result.error())); + } + if (result->has_value()) { + const auto& chunk = result->value(); + const auto expected_offset = static_cast(next_sequence_) * grain_; + if (chunk.sequence != next_sequence_ || chunk.row_offset != expected_offset) { + return std::unexpected("morsel pipeline: morsel identity gap or reordering"); + } + ++next_sequence_; + return result; + } + if (next_sequence_ != expected_morsels_) { + return std::unexpected("morsel pipeline: missing output morsel"); + } + return result; + } + + private: + OperatorPtr child_; + std::uint64_t expected_morsels_ = 0; + std::uint64_t next_sequence_ = 0; + std::size_t grain_ = 1; +}; + +} // namespace pipeline_executor_detail + +namespace pipeline_executor_detail { + +auto build_row_local_map_operator(const MapStep& step, OperatorPtr child, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, bool preserve_empty_morsels) + -> std::expected { + const MapKernelFactory factory = + step.factory != nullptr ? step.factory : map_kernel_factory(step.capability); + if (factory == nullptr) { + return std::unexpected("row-local map factory: unknown kernel capability"); + } + return factory(step, std::move(child), scalars, externs, exec, nullptr, preserve_empty_morsels); +} + +/// One unfused node as a step. The compatibility entry point for callers +/// outside a physical plan, which have a node and no fusion to express. +auto build_row_local_map_operator(const ir::Node& node, OperatorPtr child, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, bool preserve_empty_morsels) + -> std::expected { + const auto capability = map_kernel_capability(node); + if (!capability.has_value()) { + return std::unexpected("row-local map factory: unsupported kernel capability"); + } + const MapStep step{ + .node = &node, .capability = *capability, .factory = map_kernel_factory(*capability)}; + return build_row_local_map_operator(step, std::move(child), scalars, externs, exec, + preserve_empty_morsels); +} + +// The base of one worker's worker chain: a source the worker points at the +// morsel it just claimed. Two implementations, differing only in whether the +// morsel's rows are copied out of the shared input before the chain sees them. +class MorselSource : public Operator { + public: + /// Aim the source at rows [begin, end) of the pipeline's input. The next + /// `next()` produces exactly that morsel and then reports exhaustion, so + /// one call feeds one turn of the worker loop. + virtual void set_morsel(std::size_t begin, std::size_t end, std::uint64_t sequence) = 0; +}; + +// Gathering source: materializes the morsel, then the chain above runs over it +// exactly as the serial path does. The fallback for any pipeline whose head this +// file cannot evaluate by range. +class GatherMorselSource final : public MorselSource { + public: + explicit GatherMorselSource(const Table& input) : input_(&input) {} + + void set_morsel(std::size_t begin, std::size_t end, std::uint64_t sequence) override { + pending_ = make_morsel_chunk(*input_, begin, end, sequence); + } + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (!pending_.has_value()) { + return std::optional{}; + } + auto chunk = std::move(*pending_); + pending_.reset(); + return std::optional{std::move(chunk)}; + } + + private: + const Table* input_; + std::optional pending_; +}; + +// Range-filtering source: absorbs the pipeline's head `Filter` and evaluates its +// predicate directly over the input's rows [begin, end), so the morsel is never +// materialized. Only surviving rows are ever copied — the gather the serial +// path pays for every row is gone. +// +// This is one operator doing the work of two, so it owes both their contracts: +// the morsel identity a gathering source would have stamped (`sequence` and +// `row_offset`, which the worker loop re-checks), and the head filter's +// `preserve_empty_morsels` behaviour — an empty result is still emitted, +// because the merger indexes by sequence and a skipped morsel is a lost slot +// rather than a smaller answer. +class RangeFilterMorselSource final : public MorselSource { + public: + RangeFilterMorselSource(const Table& input, const ir::Expr* predicate, + const std::vector* project, + const ScalarRegistry* scalars) + : input_(&input), predicate_(predicate), project_(project), scalars_(scalars) {} + + void set_morsel(std::size_t begin, std::size_t end, std::uint64_t sequence) override { + pending_ = ChunkIdentity{.sequence = sequence, .row_offset = begin}; + begin_ = begin; + end_ = end; + } + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (!pending_.has_value()) { + return std::optional{}; + } + const auto identity = *pending_; + pending_.reset(); + const RowRange rows{.begin = begin_, .count = end_ - begin_}; + auto filtered = + project_ == nullptr + ? filter_table_range(*input_, *predicate_, rows, scalars_) + : filter_project_table_range(*input_, *predicate_, *project_, rows, scalars_); + if (!filtered.has_value()) { + return std::unexpected(std::move(filtered.error())); + } + return std::optional{table_to_chunk(std::move(filtered.value()), identity)}; + } + + private: + const Table* input_; + const ir::Expr* predicate_; + const std::vector* project_; + const ScalarRegistry* scalars_; + std::optional pending_; + std::size_t begin_ = 0; + std::size_t end_ = 0; +}; + +/// How a head operator is evaluated by range, when it can be. A filter head is +/// where gathering costs most — it copies rows the predicate is about to throw +/// away — and the physically fused filter/project form is included as well. +struct RangeHead { + const ir::Expr* predicate = nullptr; + const std::vector* project = nullptr; ///< null when unfused +}; + +/// The head's range form, or nullopt when it has to be built above a gathered +/// morsel instead. +/// +/// Two things disqualify a head: +/// +/// - A predicate that is not `is_range_native_expr`. Pipeline eligibility admits +/// Scalar calls, but every call still evaluates whole-table-and-slice, so +/// absorbing `abs(a) > 50` would re-run `abs` over the entire input once per +/// morsel — measured at 10x slower than serial on 20M rows. Gathering is the +/// correct choice there: the morsel is materialized once and the predicate +/// then runs over morsel-sized data. +/// - A column-less table, whose row count lives in the chunk's `logical_rows` +/// rather than in any column; only the gathering source carries that over. +[[nodiscard]] auto range_filter_head(const MapStep& step, const Table& input) + -> std::optional { + if (input.columns.empty()) { + return std::nullopt; + } + if (step.capability == MapKernelCapability::FilterGather) { + // A head absorbs a filter and, at most, the projection directly above + // it. A fused step that also carries an Update computes columns between + // the two, and absorbing the projection here would skip that + // computation -- the projection would then name a column nothing + // produced. Such a step keeps its operator chain. + if (step.fused_update != nullptr) { + return std::nullopt; + } + const auto& predicate = ir::node_cast(*step.node).predicate(); + if (!is_range_native_expr(predicate)) { + return std::nullopt; + } + // A planner-fused Project rides along exactly as the fused IR kind's + // column list does below: same head, same absorbed projection. + const std::vector* project = + step.fused_project != nullptr + ? &ir::node_cast(*step.fused_project).columns() + : nullptr; + return RangeHead{.predicate = &predicate, .project = project}; + } + if (step.capability == MapKernelCapability::FilterProjectGather && + step.update_fields == nullptr && step.filter_predicate != nullptr && + step.project_columns != nullptr) { + if (!is_range_native_expr(*step.filter_predicate)) { + return std::nullopt; + } + return RangeHead{.predicate = step.filter_predicate, .project = step.project_columns}; + } + return std::nullopt; +} + +// One worker's private copy of the pipeline's map chain. The operators are +// per-worker (they carry mutable per-chunk state); the IR nodes, registries, +// and the input table they read are shared and immutable for the pipeline's +// lifetime. +struct MorselWorkerChain { + MorselSource* source = nullptr; // owned by `chain`, re-aimed per morsel + OperatorPtr chain; +}; + +[[nodiscard]] auto build_morsel_worker_chain(const std::vector& operators, + const Table& input, const ScalarRegistry* scalars, + const ExternRegistry* externs, + const ExecutionContext& exec, + const JoinProbeFactory* probe_head = nullptr) + -> std::expected { + // A qualifying head is absorbed into the source rather than built as an + // operator above it — same output, without materializing the morsel first. + // Not available under a probe head: that optimization reads the first + // operator as a filter over the SOURCE's rows, and with a probe between + // them the source's rows are the probe side, not the filter's input. + std::size_t first_op = 0; + std::unique_ptr source; + if (!operators.empty() && probe_head == nullptr) { + if (auto head = range_filter_head(operators.front(), input); head.has_value()) { + source = std::make_unique(input, head->predicate, + head->project, scalars); + first_op = 1; + } + } + if (source == nullptr) { + source = std::make_unique(input); + } + + MorselWorkerChain worker{.source = source.get(), .chain = std::move(source)}; + // The probe runs first, on this worker's own morsel of the probe side, and + // every map above it runs in the same worker on the probe's output. That + // is the whole point of the fused shape: the join's output is never + // assembled as a table between the probe and the maps. + if (probe_head != nullptr) { + worker.chain = probe_head->attach(std::move(worker.chain), + /*preserve_empty_morsels=*/true); + } + for (std::size_t i = first_op; i < operators.size(); ++i) { + const MapStep& op_node = operators[i]; + // `preserve_empty_morsels` is what makes one input morsel yield exactly + // one identified output morsel — the merger indexes by sequence, so a + // silently coalesced empty result would be a lost slot, not a smaller + // answer. + auto next = build_row_local_map_operator(op_node, std::move(worker.chain), scalars, externs, + exec, true); + if (!next.has_value()) { + // The plan's step vocabulary only admits row-local map kinds. + return std::unexpected("morsel pipeline: " + next.error()); + } + worker.chain = std::move(next.value()); + } + return worker; +} +/// A bounded, sequence-ordered handoff between several producers and one +/// consumer — the one implementation of that shape in the runtime. +/// +/// Slots are addressed `sequence % window`, so a producer may run at most +/// `window` sequences ahead of the consumer and then parks; the consumer parks +/// on the slot it needs next. Both waits are wrapped in `RingWaitScope`, +/// because produced-ahead and waiting-on-workers are idle rather than work — +/// counting them as work makes a blocked worker read as a busy one and +/// overstates occupancy. +/// +/// Failure is ordered by sequence, not by arrival: the lowest-sequence failure +/// is the one reported, so the message a query returns never depends on which +/// thread lost a race. `record_fault` takes a static string and allocates +/// nothing, which is the only reporting path still available when allocation is +/// what failed. +/// +/// Producer liveness is tracked so the consumer cannot wait for a sequence that +/// is never coming: a producer that leaves for any reason — exhaustion, error, +/// exception — must call `producer_exited`, which is what turns "a worker died" +/// into an error rather than a hang. +class OrderedChunkRing { + public: + OrderedChunkRing(std::size_t window, std::size_t producers) + : window_(window == 0 ? 1 : window), + ring_(window == 0 ? 1 : window), + active_producers_(producers) {} + + OrderedChunkRing(const OrderedChunkRing&) = delete; + auto operator=(const OrderedChunkRing&) -> OrderedChunkRing& = delete; + OrderedChunkRing(OrderedChunkRing&&) = delete; + auto operator=(OrderedChunkRing&&) -> OrderedChunkRing& = delete; + ~OrderedChunkRing() = default; + + /// What a producer should do with the sequence it just claimed. + enum class Acquire : std::uint8_t { + Proceed, ///< the slot is free; produce into it + Abandon, ///< cancelled, or a lower sequence already failed + }; + + /// Park until this sequence's slot is free. Called with no lock held. + [[nodiscard]] auto acquire(std::uint64_t sequence) -> Acquire { + std::unique_lock lock(mutex_); + const RingWaitScope ring_wait; + space_.wait(lock, [&] { + return cancelled_ || sequence < released_ + window_ || + (has_error_ && error_sequence_ < sequence); + }); + // Only ever abandons sequences above the reported failure, so the + // consumer still receives everything below it. + return (cancelled_ || (has_error_ && error_sequence_ < sequence)) ? Acquire::Abandon + : Acquire::Proceed; + } + + void publish(std::uint64_t sequence, Chunk chunk) { + { + const std::scoped_lock lock(mutex_); + ring_[static_cast(sequence % window_)] = std::move(chunk); + } + ready_.notify_one(); + } + + /// Record an owned message. The caller has already built the string, so + /// taking it by value and moving it under the lock never allocates here. + void record_error(std::uint64_t sequence, std::string message) noexcept { + { + const std::scoped_lock lock(mutex_); + if (claim_failure(sequence)) { + error_owned_ = std::move(message); + error_fixed_ = nullptr; + } + } + wake_all(); + } + + /// Record a message in static storage. Allocates nothing at all, so it is + /// the only reporting path available once allocation is what failed. + void record_fault(std::uint64_t sequence, const char* message) noexcept { + { + const std::scoped_lock lock(mutex_); + if (claim_failure(sequence)) { + error_owned_.clear(); // frees, never allocates + error_fixed_ = message; + } + } + wake_all(); + } + + void producer_exited() noexcept { + { + const std::scoped_lock lock(mutex_); + --active_producers_; + } + ready_.notify_all(); + } + + /// Take the chunk at `sequence`, or nullopt when the run stopped before + /// producing it — cancelled, failed, or out of producers. The caller asks + /// `failure()` for why. + [[nodiscard]] auto take(std::uint64_t sequence) -> std::optional { + std::optional chunk; + { + std::unique_lock lock(mutex_); + const auto slot = static_cast(sequence % window_); + { + const RingWaitScope ring_wait; + ready_.wait(lock, [&] { + return ring_[slot].has_value() || cancelled_ || active_producers_ == 0 || + (has_error_ && error_sequence_ <= sequence); + }); + } + if (ring_[slot].has_value()) { + chunk = std::move(ring_[slot]); + ring_[slot].reset(); + ++released_; + } + } + if (chunk.has_value()) { + space_.notify_all(); + } + return chunk; + } + + [[nodiscard]] auto failure() const -> std::optional { + const std::scoped_lock lock(mutex_); + if (!has_error_) { + return std::nullopt; + } + return error_fixed_ != nullptr ? std::string(error_fixed_) : error_owned_; + } + + void cancel() noexcept { + { + const std::scoped_lock lock(mutex_); + cancelled_ = true; + } + wake_all(); + } + + private: + /// True if `sequence` becomes the reported failure. Lowest sequence wins, + /// so the error a query reports never depends on thread timing. + [[nodiscard]] auto claim_failure(std::uint64_t sequence) noexcept -> bool { + if (has_error_ && sequence >= error_sequence_) { + return false; + } + has_error_ = true; + error_sequence_ = sequence; + return true; + } + + void wake_all() noexcept { + ready_.notify_all(); + space_.notify_all(); + } + + std::size_t window_; + mutable std::mutex mutex_; + std::condition_variable ready_; // consumer waits for the next sequence + std::condition_variable space_; // producers wait for ring space + std::vector> ring_; + std::uint64_t released_ = 0; + std::size_t active_producers_ = 0; + bool cancelled_ = false; + // The failure channel is split so it can be written without allocating. + // `error_owned_` carries a message moved in from a producer; `error_fixed_` + // points at static storage. + bool has_error_ = false; + std::uint64_t error_sequence_ = 0; + std::string error_owned_; + const char* error_fixed_ = nullptr; +}; + +// Runtime-multithreading Phase 1: the morsel pipeline executor. +// +// Workers pull numbered morsels from one shared cursor over the immutable +// materialized input, run their own chain over each, and deposit the result in +// a bounded ring indexed by `sequence`. `next()` is the ordered merger: it +// releases results strictly in sequence order, so the operator's output is +// byte-identical to the serial chain's regardless of completion order. The ring +// is the plan's bounded in-flight queue — a worker that runs ahead of the +// consumer by a full window blocks instead of buffering the whole pipeline. +// +// Output ownership (the plan's Phase-1 allocator variable): each task owns the +// chunk it produces, and the merger's consumer moves it straight into the +// downstream `MaterializeOperator` concat. Nothing escapes into task-local +// scratch storage, so no arena ownership has to be transferred. That is the +// simplest of the strategies the plan allows and the one whose allocation +// behavior the acceptance benchmarks measure; a presized query-owned buffer +// pool is the next option if allocation shows up in those numbers. +// +// Error and cancellation determinism: a failing morsel records its error under +// the lock, keeping the *lowest* sequence, and workers abandon only morsels +// above it — so every morsel below the reported failure is still produced, and +// the error a query reports does not depend on thread timing. +class MorselPipelineOperator final : public Operator { + public: + MorselPipelineOperator(std::unique_ptr
input, std::vector workers, + std::size_t grain, std::uint64_t morsel_count, WorkerPool& pool) + : input_(std::move(input)), + workers_(std::move(workers)), + grain_(grain == 0 ? 1 : grain), + morsel_count_(morsel_count), + pool_(&pool), + ring_(std::max(workers_.size() * 2, 2), workers_.size()) {} + + ~MorselPipelineOperator() override { cancel_and_join(); } + + MorselPipelineOperator(const MorselPipelineOperator&) = delete; + auto operator=(const MorselPipelineOperator&) -> MorselPipelineOperator& = delete; + MorselPipelineOperator(MorselPipelineOperator&&) = delete; + auto operator=(MorselPipelineOperator&&) -> MorselPipelineOperator& = delete; + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (finished_) { + return std::optional{}; + } + if (!started_) { + started_ = true; + batch_ = pool_->submit(workers_.size(), [this](std::size_t id) { run_worker(id); }); + } + if (next_sequence_ >= morsel_count_) { + return finish(); + } + if (interrupt_requested()) { + return fail(interrupt_message()); + } + + std::optional chunk = ring_.take(next_sequence_); + if (chunk.has_value()) { + ++next_sequence_; + return chunk; + } + + // No chunk: the pipeline stopped early. Report why, deterministically. + // + // An interrupt outranks a recorded data error. A worker that fails at + // the moment the user hits Ctrl+C is a race, and reporting its message + // would make cancellation surface as an arbitrary query error depending + // on which thread won. The cancellation contract says such a query + // reports "interrupted", so the interrupt is checked first. + if (interrupt_requested()) { + return fail(interrupt_message()); + } + if (auto failure = ring_.failure(); failure.has_value()) { + return fail(std::move(*failure)); + } + return fail("morsel pipeline: missing output morsel"); + } + + private: + // Whatever happens to a worker — normal exhaustion, a recorded error, or an + // exception — it must stop counting as active and must wake the merger. + // Skipping this on any path leaves the consumer waiting for a sequence that + // is never coming, which is a hang rather than an error. + void worker_exited() noexcept { ring_.producer_exited(); } + + void run_worker(std::size_t worker_id) noexcept { + // Cleanup runs however this scope is left, so no path can leave the + // consumer waiting on a worker that is gone. + // One-shot scope guard: aggregate-initialised, never copied or moved. + // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions) + struct ExitGuard { + MorselPipelineOperator* self; + ~ExitGuard() { self->worker_exited(); } + } const guard{this}; + + std::uint64_t sequence = 0; + try { + run_worker_loop(worker_id, sequence); + } catch (const std::exception& error) { + // An exception is not part of the operator protocol (evaluation + // reports failure through `expected`), so it is something + // unplanned — an allocation failure while materializing a morsel, + // say. Convert it to a sequence-tagged pipeline error so it obeys the + // same lowest-sequence determinism as any other failure, rather + // than unwinding through a pool thread. + // + // Composing that message allocates, and the exception this handler + // most expects is `bad_alloc` — so the detailed message is + // best-effort, with an allocation-free fallback underneath it. + // Throwing from here would terminate the process, since this + // function is noexcept precisely so a worker cannot unwind into the + // pool. `what()` cannot be stored: it dies with the exception. + try { + ring_.record_error( + sequence, "morsel pipeline: worker exception: " + std::string(error.what())); + } catch (...) { + ring_.record_fault(sequence, + "morsel pipeline: worker exception (no memory to report it)"); + } + } catch (...) { + ring_.record_fault(sequence, "morsel pipeline: worker threw a non-standard exception"); + } + } + + void run_worker_loop(std::size_t worker_id, std::uint64_t& claimed) { + auto& worker = workers_[worker_id]; + const std::size_t rows = input_->rows(); + while (true) { + const std::uint64_t sequence = cursor_.fetch_add(1, std::memory_order_relaxed); + if (sequence >= morsel_count_) { + break; + } + // Published so an exception thrown below is attributed to the + // morsel that was in flight, not to sequence 0. + claimed = sequence; + // Backpressure: this morsel's slot is only free once the consumer + // has released the morsel `window` ahead of it. + if (ring_.acquire(sequence) == OrderedChunkRing::Acquire::Abandon) { + break; + } + if (interrupt_requested()) { + ring_.cancel(); + break; + } + + const auto [begin, end] = morsel_row_range(rows, grain_, sequence); + worker.source->set_morsel(begin, end, sequence); + auto produced = worker.chain->next(); + + if (!produced.has_value()) { + ring_.record_error(sequence, std::move(produced.error())); + break; + } + if (!produced->has_value()) { + ring_.record_fault(sequence, "morsel pipeline: worker produced no output morsel"); + break; + } + Chunk out = std::move(**produced); + if (out.sequence != sequence || out.row_offset != begin) { + ring_.record_fault(sequence, "morsel pipeline: morsel identity gap or reordering"); + break; + } + ring_.publish(sequence, std::move(out)); + } + } + + // Called from the destructor, so nothing here may throw: an escaping + // exception during destruction terminates the process. Worker bodies are + // already noexcept and convert failures into pipeline errors, so there is + // nothing for `wait()` to rethrow — this guards the path regardless. + void cancel_and_join() noexcept { + try { + ring_.cancel(); + batch_.wait(); + } catch (...) { // NOLINT(bugprone-empty-catch) + // Nothing left to report: the caller is either unwinding or has + // already chosen the message it will return. + } + } + + // Drain the pipeline cleanly at EOF, then check the per-worker chains really + // are exhausted: a chain still holding a suppressed schema carrier would + // mean a morsel was coalesced away rather than emitted. + [[nodiscard]] auto finish() -> std::expected, std::string> { + finished_ = true; + batch_.wait(); + // Same precedence as `next()`: a cancelled run reports cancellation + // even if a worker also failed on its way out. + if (interrupt_requested()) { + return std::unexpected(interrupt_message()); + } + if (auto failure = ring_.failure(); failure.has_value()) { + return std::unexpected(std::move(*failure)); + } + for (auto& worker : workers_) { + auto trailing = worker.chain->next(); + if (!trailing.has_value()) { + return std::unexpected(std::move(trailing.error())); + } + if (trailing->has_value()) { + return std::unexpected("morsel pipeline: unexpected trailing morsel"); + } + } + return std::optional{}; + } + + [[nodiscard]] auto fail(std::string message) + -> std::expected, std::string> { + finished_ = true; + cancel_and_join(); + return std::unexpected(std::move(message)); + } + + // `input_` is declared first so it outlives `workers_`: the chains read it + // through raw pointers, and the batch is joined before any member is + // destroyed. + std::unique_ptr
input_; + std::vector workers_; + std::size_t grain_ = 1; + std::uint64_t morsel_count_ = 0; + WorkerPool* pool_; + + std::atomic cursor_{0}; + + // The ordered handoff between the workers and this operator's `next()`. + OrderedChunkRing ring_; + + std::uint64_t next_sequence_ = 0; + bool started_ = false; + bool finished_ = false; + WorkerPool::Batch batch_; +}; + +// Runtime-multithreading Phase 2: the two-phase parallel filter. +// +// What the ordered merger above cannot remove is the merge itself. Each worker +// materializes its morsel's surviving rows, and `MaterializeOperator` then +// copies all of them again into one table — so a filter pipeline copies its +// output twice where the serial path copies it once. That is why morsel parallelism wins +// track OUTPUT size rather than input size: a selective predicate wins easily, +// and a bulk one loses no matter how much input work is parallelized. +// +// A filter cannot simply presize its output and skip the merge, because its +// cardinality is data-dependent — nobody knows where morsel 7's rows belong +// until morsels 0-6 have been counted. So run the filter in two passes: +// +// Phase A every morsel evaluates the predicate and packs its surviving +// rows into keep words, in parallel. Only the counts matter after. +// (serial) an exclusive prefix sum over those counts gives each morsel the +// row — and, for string columns, the byte — where its output +// begins. The output is then allocated ONCE, at exactly the +// final size. +// Phase B every morsel gathers its rows straight into that shared output +// at its own offset, in parallel. The slices are disjoint, so no +// locking is needed and nothing is copied twice. +// +// The result is emitted as ONE chunk, which `MaterializeOperator` moves instead +// of concatenating. Ordering is structural — a morsel's rows land at its +// prefix-sum offset — so there is no ring, no merger, and the output is +// byte-identical to the serial filter's. +// +// What it costs: phase A's keep words are held for every morsel at once, which +// is one bit per input row (2.5MB for 20M rows), and phase B re-walks them. +// Neither re-evaluates the predicate. +// +// Writing into disjoint output ROWS is only disjoint in MEMORY for columns +// storing at least one addressable unit per row. `Column` and validity +// bitmaps pack 64 rows to a word, so two morsels meeting mid-word touch the +// same word; `gather_selection_into` resolves that with the shared-word rule +// (see `SharedBitWords` in filter.cpp) rather than excluding those columns. +// `filter_gather_is_thread_safe` remains as the allowlist that keeps a future +// column kind out until someone has checked it. +// +// Note a 64-row-aligned grain would NOT have made those columns safe, which is +// the tempting shortcut: an output offset is the prefix sum of POPCOUNTS, not +// of morsel sizes, so a morsel keeping 37 of its 64 rows already leaves the +// next one starting mid-word. Grain only aligns the SOURCE read, and reads +// never race. +class TwoPhaseFilterOperator final : public Operator { + public: + TwoPhaseFilterOperator(std::unique_ptr
input, const ir::Expr& predicate, + bool fused_project, std::vector tail, + const ScalarRegistry* scalars, FilterOutputLayout layout, + std::size_t grain, std::uint64_t morsel_count, std::size_t workers, + WorkerPool& pool) + : input_(std::move(input)), + predicate_(&predicate), + fused_project_(fused_project), + tail_(std::move(tail)), + scalars_(scalars), + layout_(std::move(layout)), + grain_(grain == 0 ? 1 : grain), + morsel_count_(morsel_count), + workers_(workers), + pool_(&pool) { + selections_.resize(static_cast(morsel_count_)); + row_at_.assign(static_cast(morsel_count_), 0); + } + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (done_) { + return std::optional{}; + } + done_ = true; + auto table = run(); + if (!table.has_value()) { + return std::unexpected(std::move(table.error())); + } + // Sequence 0 / row_offset 0: this operator emits the pipeline's whole + // output at once, so it is trivially the first and only morsel. + return std::optional{table_to_chunk(std::move(table.value()), ChunkIdentity{})}; + } + + private: + [[nodiscard]] auto run() -> std::expected { + const std::size_t n_cols = layout_.output.columns.size(); + const bool has_strings = std::ranges::any_of(layout_.src_of_dst, [&](std::size_t src) { + return std::holds_alternative>(*input_->columns[src].column); + }); + if (has_strings) { + chars_at_.assign(static_cast(morsel_count_), + std::vector(n_cols, 0)); + } + + if (auto failure = run_over_morsels([this](std::uint64_t sequence) { phase_a(sequence); }); + failure.has_value()) { + return std::unexpected(std::move(*failure)); + } + + // Exclusive prefix sums: each morsel's counts become its offsets, and + // the running totals become the output's exact size. Serial on purpose + // — it is O(morsels), not O(rows). + std::size_t rows_total = 0; + for (std::size_t m = 0; m < selections_.size(); ++m) { + row_at_[m] = rows_total; + rows_total += selections_[m].kept; + } + std::vector chars_total(n_cols, 0); + if (has_strings) { + for (std::size_t d = 0; d < n_cols; ++d) { + for (auto& per_morsel : chars_at_) { + const std::size_t count = per_morsel[d]; + per_morsel[d] = chars_total[d]; + chars_total[d] += count; + } + } + } + + presize_filter_output(layout_.output, *input_, layout_.src_of_dst, rows_total, chars_total); + + if (auto failure = run_over_morsels([this](std::uint64_t sequence) { phase_b(sequence); }); + failure.has_value()) { + return std::unexpected(std::move(*failure)); + } + + // Identical rule to the serial filter: a row-local filter preserves + // order and time index; a fused projection keeps each only when its + // column survives. + apply_table_properties( + layout_.output, TableProperties::derive( + table_properties_of(*input_), + [&](const std::string& name) -> KeyFate { + return (!fused_project_ || layout_.output.index.contains(name)) + ? KeyFate::kept(name) + : KeyFate::dropped(); + }, + RowTransform::Subset)); + + // Metadata-only operators above the filter run ONCE over the finished + // output rather than per morsel. `project_table` / `rename_table` build + // their result with `add_column_shared` — zero rows copied, O(columns) + // — so running them serially here costs nothing, while routing the + // chain through the ordered merger to parallelize them costs the whole + // merge copy. Measured: `filter … rename` at 93% selectivity was 2.3x + // SLOWER than serial on the merger. + // + // Applying them to the concatenated output is equivalent to applying + // them per morsel because neither reads a row; and because these are + // the same two functions the serial path calls, the ordering and + // time-index rules cannot diverge from it either. + Table result = std::move(layout_.output); + for (const ir::Node* node : tail_) { + auto next = apply_metadata_only(*node, result); + if (!next.has_value()) { + return std::unexpected(std::move(next.error())); + } + result = std::move(next.value()); + } + return result; + } + + [[nodiscard]] static auto apply_metadata_only(const ir::Node& node, const Table& input) + -> std::expected { + switch (node.kind()) { + case ir::NodeKind::Project: + return project_table(input, ir::node_cast(node).columns()); + case ir::NodeKind::Rename: + return rename_table(input, ir::node_cast(node).renames()); + default: + // The pipeline builder only admits `is_metadata_only_node` kinds + // into `tail_`, so reaching this means the two have drifted. + invariant_violation("two-phase filter: non-metadata operator in the tail"); + } + } + + void phase_a(std::uint64_t sequence) { + const auto rows = morsel_range(sequence); + auto selection = + compute_filter_selection(*input_, *predicate_, scalars_, rows, /*row_limit=*/0); + if (!selection.has_value()) { + record_error(sequence, std::move(selection.error())); + return; + } + auto& slot = selections_[static_cast(sequence)]; + slot = std::move(selection.value()); + if (!chars_at_.empty()) { + count_selected_chars(*input_, layout_.src_of_dst, slot, rows, + chars_at_[static_cast(sequence)]); + } + } + + void phase_b(std::uint64_t sequence) { + const auto index = static_cast(sequence); + gather_selection_into( + layout_.output, *input_, layout_.src_of_dst, selections_[index], morsel_range(sequence), + GatherDest{.row = row_at_[index], + .char_base = chars_at_.empty() ? nullptr : &chars_at_[index]}); + } + + [[nodiscard]] auto morsel_range(std::uint64_t sequence) const -> RowRange { + const auto [begin, end] = morsel_row_range(input_->rows(), grain_, sequence); + return RowRange{.begin = begin, .count = end - begin}; + } + + // Run `body` over every morsel across the pool and join. A failure is + // reported with the LOWEST morsel sequence, the same determinism rule the + // ordered merger uses, so which morsel's error a query reports never + // depends on thread timing. Workers abandon only morsels ABOVE a recorded + // failure, so no morsel below it is skipped. + template + [[nodiscard]] auto run_over_morsels(const Body& body) -> std::optional { + reset_failure(); + std::atomic cursor{0}; + { + auto batch = pool_->submit(workers_, [&](std::size_t) noexcept { + while (true) { + const std::uint64_t sequence = cursor.fetch_add(1, std::memory_order_relaxed); + if (sequence >= morsel_count_) { + return; + } + if (failure_below(sequence) || interrupt_requested()) { + return; + } + // A worker may not unwind into the pool, so an unplanned + // exception becomes a sequence-tagged error like any other. + // The message itself allocates and the exception most + // expected here is bad_alloc, so there is an + // allocation-free fallback underneath it. + try { + body(sequence); + } catch (const std::exception& error) { + try { + record_error(sequence, "parallel filter: worker exception: " + + std::string(error.what())); + } catch (...) { + record_fault(sequence, + "parallel filter: worker exception (no memory to " + "report it)"); + } + } catch (...) { + record_fault(sequence, + "parallel filter: worker threw a non-standard exception"); + } + } + }); + batch.wait(); + } + // Same precedence as the ordered merger: an interrupt outranks a + // recorded data error, so a worker failing as Ctrl+C arrives still + // reports cancellation rather than an arbitrary error. + if (interrupt_requested()) { + return interrupt_message(); + } + const std::scoped_lock lock(mutex_); + if (!has_error_) { + return std::nullopt; + } + return error_fixed_ != nullptr ? std::string(error_fixed_) : error_owned_; + } + + void reset_failure() noexcept { + const std::scoped_lock lock(mutex_); + has_error_ = false; + error_fixed_ = nullptr; + error_owned_.clear(); + } + + [[nodiscard]] auto failure_below(std::uint64_t sequence) noexcept -> bool { + const std::scoped_lock lock(mutex_); + return has_error_ && error_sequence_ < sequence; + } + + void record_error(std::uint64_t sequence, std::string message) { + const std::scoped_lock lock(mutex_); + if (claim_failure(sequence)) { + error_owned_ = std::move(message); + error_fixed_ = nullptr; + } + } + + void record_fault(std::uint64_t sequence, const char* message) noexcept { + const std::scoped_lock lock(mutex_); + if (claim_failure(sequence)) { + error_owned_.clear(); // frees, never allocates + error_fixed_ = message; + } + } + + [[nodiscard]] auto claim_failure(std::uint64_t sequence) noexcept -> bool { + if (has_error_ && sequence >= error_sequence_) { + return false; + } + has_error_ = true; + error_sequence_ = sequence; + return true; + } + + // `input_` is declared first so it outlives everything reading it. + std::unique_ptr
input_; + const ir::Expr* predicate_; + bool fused_project_ = false; + /// Metadata-only operators above the filter, source-to-sink, applied once + /// to the finished output. Every element is `is_metadata_only_node`. + std::vector tail_; + const ScalarRegistry* scalars_; + FilterOutputLayout layout_; + std::size_t grain_ = 1; + std::uint64_t morsel_count_ = 0; + std::size_t workers_ = 0; + WorkerPool* pool_; + + // Written by phase A, read by phase B. Every element is touched by exactly + // one worker (indexed by its own morsel sequence), so these need no lock — + // the join between the phases is the synchronization. + std::vector selections_; + std::vector row_at_; + std::vector> chars_at_; // empty when no string column + + std::mutex mutex_; + bool has_error_ = false; + std::string error_owned_; + const char* error_fixed_ = nullptr; + std::uint64_t error_sequence_ = 0; + + bool done_ = false; +}; + +// How many workers a pipeline of `morsel_count` morsels over `rows` rows should +// run on: 0 means "stay on the serial morsel chain". +// +// This is the plan's grain-size serial threshold. Below it, task dispatch, +// ring synchronization, and the merge cost more than the map they parallelize — +// cache-resident work should not pay for threads. A single morsel is serial by +// definition, and a one-thread budget means the caller asked for serial. +/// Whether this input is worth morselizing at all — a *different* question from +/// how many workers it deserves, and conflating the two is a trap worth naming. +/// +/// A "refused" pipeline used to mean a serial sweep of morsels, which still pays +/// per-morsel materialization and the merge concat. So refusing by dropping the +/// worker count made a small query **slower than never forming a pipeline**: +/// measured 100ms against 36ms for the plain serial path, and it got worse once +/// the grain was derived, because that turned 2 morsels into 32. When the +/// answer is no, the input has to run as ONE whole-table chunk. +/// +/// Two thresholds, because a pipeline's cost has two dimensions. Rows alone +/// cannot express it: 131,072 rows won at 6 columns and lost at 2 on the very +/// same predicate, and every row threshold puts those on the same side. +[[nodiscard]] auto is_worth_morselizing(const ExecutionContext& exec, std::size_t rows, + std::size_t columns) -> bool { + if (rows < exec.parallel_min_rows) { + return false; + } + return exec.parallel_min_cells == 0 || columns == 0 || + rows * columns >= exec.parallel_min_cells; +} + +[[nodiscard]] auto morsel_worker_count(const ExecutionContext& exec, std::uint64_t morsel_count) + -> std::size_t { + if (morsel_count < 2 || !exec.can_fan_out()) { + return 0; + } + // Past the parallel gate: consulting the pool here is free of the + // construct-before-declining hazard because a parallel query has already + // built it (or is about to, on its first fan-out). + const std::size_t pool_size = process_worker_pool().size(); + const std::size_t budget = exec.compute_budget(); + const std::size_t workers = + std::min({budget, pool_size, static_cast(morsel_count)}); + return workers < 2 ? 0 : workers; +} + +// Build one eligible row-local parallel-map chain as a morsel pipeline: materialize its +// input subtree once, then run the chain over morsels of that table instead of +// one whole-table chunk. The operators are ordered source-to-sink. +// +// Two executors, one morsel model. A large input fans out across the worker +// pool and is reassembled by `MorselPipelineOperator`'s ordered merger; a small +// one (or a single-threaded budget) runs the same morsels serially through a +// `PartitionedTableSource`, where `MaterializeOperator`'s in-order concat is +// the trivially ordered merger. Both stamp and check the same morsel identity, +// so both are byte-identical to the plain serial chain — which is exactly what +// lets the threshold move without changing an answer. +// +// LOAD-BEARING INVARIANT — materialize before fan-out. The input subtree is +// executed to a `Table` here, on this thread, and every morsel source below +// takes that finished table by reference. That is what makes a deferred/lazy +// source safe in a pipeline: its decode runs exactly once, serially, before any +// worker exists, so neither `LazyTable::cache_` nor a plugin's `decode_` +// closure is ever touched concurrently. It is why `build_operator`'s seam no +// longer screens pipelines for deferred sources. +// +// The morsel sources all take `const Table&`, so the invariant is enforced by +// their signatures rather than by a check. Streaming a source's morsels +// directly into workers would mean handing them something other than a +// finished table — at which point the LazyTable synchronization contract +// applies in full and eligibility has to be re-established. +// +// `scan_pipeline_worker_count` and `build_pipelined_scan` are defined below and +// declared in `pipeline_executor_internal.hpp`; both are consulted by the run +// builder, which decides its own source strategy. + +} // namespace pipeline_executor_detail + +// Internal linkage to match the forward declarations beside the join, which +// sit inside this TU's anonymous namespace; the definitions must live down +// here because they use the morsel executor, which is defined below the join. +namespace pipeline_executor_detail { + +/// Build a streaming join here and take its probe, so a map pipeline above it +/// can run that probe at the head of its own worker chains. +/// +/// Returns an empty optional -- not an error -- for every join this cannot +/// fuse: a materializing one, a semi/anti one, a deferred probe scan (whose +/// right subtree must be interpreted by the join itself, after it publishes +/// build-side bounds), and any orientation that leaves no materialized probe +/// side. The caller then materializes the join's output as it always did. +/// +/// Narrow on purpose. The point is to establish that two pipelines meeting at +/// a barrier can be built and can produce the right answer; widening the +/// shapes is cheap once that is true, and pointless before. +auto try_take_join_probe(const ir::Node& node, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected, std::string> { + if (std::getenv("IBEX_PROBE_MORSELS") == nullptr) { + return std::optional{}; + } + const physical::Plan plan = physical::plan_physical(node, registry, externs); + const physical::JoinPlan& jp = plan.join; + if (!jp.describes || jp.strategy != physical::JoinStrategy::StreamingProbe || + (jp.branch != physical::JoinBranch::SingleKeyInner && + jp.branch != physical::JoinBranch::PairIntInner)) { + return std::optional{}; + } + const auto& join = ir::node_cast(node); + if (deferred_probe_scan_of(*join.children()[1], exec).scan != nullptr) { + return std::optional{}; + } + + auto left_op = build_operator(*join.children()[0], registry, scalars, externs, exec, model_out); + if (!left_op.has_value()) { + return std::unexpected(std::move(left_op.error())); + } + auto right = + materialize_row_local(*join.children()[1], registry, scalars, externs, exec, model_out); + if (!right.has_value()) { + return std::unexpected(std::move(right.error())); + } + return take_fusible_join_probe(std::move(left_op.value()), std::move(right.value()), + &join.keys(), exec, join.suffix(), &join.pending_order(), + physical_executor_detail::resolved_join_parallelism(exec)); +} + +auto probe_morsel_workers(const Table& input, const ExecutionContext& exec) -> std::size_t { + // OPT-IN, and the measurement is why. On its own this is a LOSS: the + // probe already fans out inside one chunk (`probe_ranges_parallel`) with + // no copy, and morselizing replaces that with a per-morsel gather plus an + // ordered merge. Measured at 8 cores, interleaved, median of 15: + // q05 +1.7%, q09 +6.8% -- the only two PDS-H queries where it fires. + // + // It is wired anyway because the gather is not what this shape is for. + // A probe morsel becomes worth its gather when the filters and + // projections above the join run in the SAME chain, so the join's output + // is never materialized between them. That needs the probe admitted into + // the plan's step vocabulary, which is the next piece; this is the half + // that had to work first. This remains opt-in: the former "22/22" claim is + // stale, and SF4 q09 currently stalls in both the pre-extraction baseline + // and this tree when the knob is enabled. Resolve that independently + // before admitting this shape by default. + if (std::getenv("IBEX_PROBE_MORSELS") == nullptr) { + return 0; + } + if (!exec.can_fan_out() || on_worker_pool_thread()) { + return 0; + } + if (!is_worth_morselizing(exec, input.rows(), input.columns.size())) { + return 0; + } + const std::size_t grain = morsel_grain(exec, input.rows()); + return morsel_worker_count(exec, partitioned_morsel_count(input, grain)); +} + +auto build_probe_morsel_pipeline(Table input, const JoinProbeFactory& probe, std::size_t workers, + const ExecutionContext& exec) + -> std::expected { + auto owned = std::make_unique
(std::move(input)); + const std::size_t grain = morsel_grain(exec, owned->rows()); + const auto expected_morsels = partitioned_morsel_count(*owned, grain); + if (exec.parallel_stats != nullptr) { + exec.parallel_stats->parallel_pipelines.fetch_add(1, std::memory_order_relaxed); + exec.parallel_stats->morsels.fetch_add(expected_morsels, std::memory_order_relaxed); + } + + static const std::vector no_steps; + std::vector chains; + chains.reserve(workers); + for (std::size_t i = 0; i < workers; ++i) { + // The chain is source + probe. Each worker gets its OWN `JoinProbe` -- + // its scratch vectors and its per-chunk categorical head table are + // per-worker state -- and they share one build, because the index and + // the build side are both `shared_ptr` and have been since + // `f6a1a632` and `6df9a966`. Copying the probe is what that ownership + // was for. + auto worker = build_morsel_worker_chain(no_steps, *owned, nullptr, nullptr, exec, &probe); + if (!worker.has_value()) { + return std::unexpected(std::move(worker.error())); + } + chains.push_back(std::move(worker.value())); + } + return std::make_unique(std::move(owned), std::move(chains), grain, + expected_morsels, process_worker_pool()); +} + +} // namespace pipeline_executor_detail + +namespace pipeline_executor_detail { + +/// The steps of a plan's parallel prefix, ordered source-to-sink. A plan +/// records steps sink-first; every executor here composes bottom-up. +auto parallel_pipeline_operators(const physical::Plan& plan) -> std::vector { + std::vector operators; + operators.reserve(plan.parallel_step_count()); + for (std::size_t i = plan.parallel_end; i > plan.parallel_begin; --i) { + operators.push_back(plan.steps[i - 1]); + } + return operators; +} + +/// Run a physical map pipeline over morsels. The plan says which steps may run +/// in parallel (`parallel_steps`) and what feeds them (`parallel_input_node`); +/// this builds that input, materializes it, and executes the prefix over its +/// morsels. It is the pipeline's parallel *mode*, not a separate executor with +/// its own idea of what is eligible. +/// +/// `steps` inside a plan are sink-first; the operators here run source-to-sink, +/// so the prefix is reversed once, on the build thread. +auto build_map_pipeline_parallel(const physical::Plan& plan, const TableRegistry& registry, + const ScalarRegistry* scalars, const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected { + const std::vector operators = parallel_pipeline_operators(plan); + const ir::Node* input_node = physical::parallel_input_node(plan); + if (input_node == nullptr) { + return std::unexpected("map pipeline: parallel mode without an input"); + } + + // Source strategy, decided here because it is a property of this run's + // input rather than of the query's root. A decomposable deferred scan can + // feed the run one unit at a time -- decode and maps in the same worker + // task, the ordered ring feeding whatever is above -- instead of being + // decoded whole and morselized. Probe scans keep their join-owned dynamic + // filter timing (a null filter slot is what distinguishes them) and so do + // not stream here. + if (exec.stream_scans && input_node->kind() == ir::NodeKind::Scan) { + const auto& scan = ir::node_cast(*input_node); + if (!registry.contains(scan.source_name())) { + if (const auto* deferred = exec.deferred_scan(scan.source_name()); + deferred != nullptr && deferred->filter == nullptr) { + auto units = deferred_scan_units(*deferred); + if (units.size() > 1 && scan_pipeline_worker_count(units.size()) >= 2) { + return build_pipelined_scan(operators, true, *deferred, std::move(units), + scalars, externs, exec); + } + } + } + } + + // Fused probe: when this pipeline's input is a streaming join, take its + // build and its probe side and run the probe at the head of every worker + // chain, instead of materializing the join's OUTPUT and morselizing that. + // The probe side becomes the morsel source, so the join's output is never + // assembled as a table at all -- it is produced a morsel at a time by the + // same worker that then runs the maps over it. Two pipelines meeting at a + // barrier, which is the shape this plan has been working toward. + std::optional fused_probe; + std::unique_ptr
owned; + if (input_node->kind() == ir::NodeKind::Join) { + auto fused = try_take_join_probe(*input_node, registry, scalars, externs, exec, model_out); + if (!fused.has_value()) { + return std::unexpected(std::move(fused.error())); + } + if (fused->has_value()) { + owned = std::make_unique
(std::move((*fused)->probe_side)); + fused_probe = std::move((*fused)->probe); + } + } + if (owned == nullptr) { + auto input_op = build_operator(*input_node, registry, scalars, externs, exec, model_out); + if (!input_op.has_value()) { + return std::unexpected(std::move(input_op.error())); + } + auto input_tbl = materialize_operator(std::move(input_op.value())); + if (!input_tbl.has_value()) { + return std::unexpected(std::move(input_tbl.error())); + } + owned = std::make_unique
(std::move(input_tbl.value())); + } + const std::size_t grain = morsel_grain(exec, owned->rows()); + const auto expected_morsels = partitioned_morsel_count(*owned, grain); + // Morselize only when the work would actually fan out. Splitting earns its + // cost by having several workers share it; with fewer than two the split, + // the per-morsel gather and the merge concat are all paid for parallelism + // that was already ruled out, so the whole-table chunk below is the right + // shape. + // + // No time claim is attached to this: measured on its own it moved nothing + // (`morsels` 92 -> 0 on q12/q14, wall time unchanged), because by the time + // this runs the input has already been materialized. The run's real cost at + // a budget of one is that materialize, and it is declined at the + // construction seam in `build_physical_map_step` instead. What this does + // earn is an honest counter -- see below -- and not doing work whose only + // consumer is a worker that will not exist. + const std::size_t worker_count = + is_worth_morselizing(exec, owned->rows(), owned->columns.size()) + ? morsel_worker_count(exec, expected_morsels) + : 0; + const bool morselize = worker_count >= 2; + if (exec.parallel_stats != nullptr) { + auto& stats = *exec.parallel_stats; + (morselize ? stats.parallel_pipelines : stats.serial_pipelines) + .fetch_add(1, std::memory_order_relaxed); + if (morselize) { + // Only count morsels that are actually formed. Reporting the + // would-be split for a run that takes the whole-table chunk is what + // made this cost invisible: `morsels=115` on a query with + // `parallel=0` reads as work done, not as work paid for and thrown + // away. + stats.morsels.fetch_add(expected_morsels, std::memory_order_relaxed); + } + } + + if (worker_count >= 2) { + const auto head = (operators.empty() || fused_probe.has_value()) + ? std::nullopt + : range_filter_head(operators.front(), *owned); + if (exec.parallel_stats != nullptr && head.has_value()) { + exec.parallel_stats->range_heads.fetch_add(1, std::memory_order_relaxed); + } + + // A range-native filter can skip the merger entirely by presizing its + // output — see TwoPhaseFilterOperator. Anything above it must be + // metadata-only: a row-touching operator would need the per-morsel + // chunks the two-phase form does not produce, but Project and Rename + // copy no rows and so are simply run once over the finished output. + const auto tail = std::span{operators}.subspan(head.has_value() ? 1 : 0); + if (head.has_value() && std::ranges::all_of(tail, [](const MapStep& step) { + return is_metadata_only_node(step.node->kind()); + })) { + auto layout = build_filter_output_layout(*owned, head->project); + // A missing projected column is left to the ordered merger below, + // which reports it through the normal evaluation path. + if (layout.has_value() && filter_gather_is_thread_safe(*owned, layout->src_of_dst)) { + if (exec.parallel_stats != nullptr) { + exec.parallel_stats->two_phase_filters.fetch_add(1, std::memory_order_relaxed); + } + // The tail is metadata-only by the check above, so no step in + // it carries a fused partner; its nodes are the whole story. + std::vector tail_nodes; + tail_nodes.reserve(tail.size()); + for (const MapStep& step : tail) { + tail_nodes.push_back(step.node); + } + return std::make_unique( + std::move(owned), *head->predicate, head->project != nullptr, + std::move(tail_nodes), scalars, std::move(layout.value()), grain, + expected_morsels, worker_count, process_worker_pool()); + } + } + + std::vector workers; + workers.reserve(worker_count); + for (std::size_t i = 0; i < worker_count; ++i) { + auto worker = + build_morsel_worker_chain(operators, *owned, scalars, externs, exec, + fused_probe.has_value() ? &*fused_probe : nullptr); + if (!worker.has_value()) { + return std::unexpected(std::move(worker.error())); + } + workers.push_back(std::move(worker.value())); + } + return std::make_unique(std::move(owned), std::move(workers), grain, + expected_morsels, process_worker_pool()); + } + + if (!morselize) { + // Too little work to be worth splitting: run the chain over one + // whole-table chunk. This is the plain serial path — same map + // operators, same `preserve_empty_morsels = false`, one chunk in and + // one chunk out — so it costs exactly what not forming a pipeline costs. + // Morselizing here instead would add a per-morsel gather and a merge + // concat to buy parallelism that was already judged not worth having. + OperatorPtr serial = make_table_source(std::move(*owned)); + if (fused_probe.has_value()) { + // Not worth morselizing, but the probe was already taken from the + // join and there is nothing to give it back to: run it here, over + // the whole probe side, with the maps above it as before. + serial = fused_probe->attach_move(std::move(serial)); + } + for (const MapStep& op_node : operators) { + auto next = build_row_local_map_operator(op_node, std::move(serial), scalars, externs, + exec, false); + if (!next.has_value()) { + return std::unexpected("morsel pipeline: " + next.error()); + } + serial = std::move(next.value()); + } + return serial; + } + + OperatorPtr chain = std::make_unique(*owned, grain); + for (const MapStep& op_node : operators) { + auto next = + build_row_local_map_operator(op_node, std::move(chain), scalars, externs, exec, true); + if (!next.has_value()) { + // The plan's step vocabulary only admits row-local map kinds. + return std::unexpected("morsel pipeline: " + next.error()); + } + chain = std::move(next.value()); + } + + chain = std::make_unique(std::move(chain), expected_morsels, grain); + return std::make_unique(std::move(owned), std::move(chain)); +} + +/// Streams a deferred lazy scan one source unit at a time instead of decoding +/// the whole source and handing it over as a single chunk. +/// +/// This is Phase 1 of `plans/pipelined-execution-plan.md`. The decode it +/// performs is the same decode `materialize_deferred_scan` performs, with the +/// same pushdowns — projection, static conjuncts, the dynamic key membership +/// filter, and both fused scans, all restricted to the unit rather than +/// declined (see `LazyTable::project_where_unit`). What changes is only that +/// the rows arrive in pieces, which is the precondition for anything above the +/// scan ever running concurrently with it. +/// +/// The plan is fixed once, at construction, for the reason `DeferredScanPlan` +/// documents: re-reading the shared filter slot per unit could apply to unit 3 +/// a bound that units 0-2 never saw. +/// +/// **Phase 2 (concurrent units).** Units are decoded a WINDOW at a time on +/// worker threads rather than one after another, and the window after the one +/// being served is already decoding. Phase 1 measured why: decoding units +/// serially cut total work (pool work on q01 fell 234ms -> 125ms) but raised +/// 8-core wall, because the pool saw one short burst per unit with a serial +/// phase between and occupancy fell to 0.14. Nothing was too small to +/// parallelize — there was just never more than one unit's worth of work +/// available at a time. +/// +/// Decoding a unit on a worker is safe, and specifically so: +/// +/// * `LazyTable::acquire_reader` hands each concurrent acquisition its OWN +/// reader product, under a mutex, and a product owns all the mutable +/// decoder state. That is exactly what the reader pool was built for. +/// * `project_where_unit` never WRITES `cache_` — a unit holds a fragment of +/// a column, so it must not — and concurrent reads of it are fine. This is +/// load-bearing now, not just a correctness nicety: routing any part of the +/// unit path back through `project()`, which does cache, would turn this +/// into a data race. +/// * Every inner parallel path (`parallel_readers`, `for_row_ranges`, +/// `filter_selection`) checks `on_worker_pool_thread()` and runs serial +/// inside a task, so the outer window is the only level of parallelism and +/// nothing submits from a worker into a saturated pool. +/// +/// Ordering is preserved exactly: workers claim units from a shared cursor and +/// write only their own slot, and chunks are served in unit order with +/// `sequence` / `row_offset` assigned on the calling thread. The categorical +/// remap also stays on the calling thread, in unit order, because it folds each +/// chunk into a dictionary shared with every earlier one. +class DeferredScanSourceOperator final : public Operator { + public: + DeferredScanSourceOperator(const DeferredScan& scan, std::vector units, + const ExecutionContext& exec) + : scan_(&scan), + plan_(plan_deferred_scan(scan)), + units_(std::move(units)), + exec_(&exec), + window_(unit_window(exec)) {} + + [[nodiscard]] auto next() -> std::expected, std::string> override { + while (true) { + if (served_ < ready_.size()) { + auto& slot = ready_[served_++]; + if (!slot.has_value()) { + return std::unexpected(std::move(slot.error())); + } + auto chunk = emit(std::move(*slot)); + if (!chunk.has_value()) { + continue; // the unit's rows were all filtered out + } + return chunk; + } + // Out of decoded units: put the next window in flight and wait for + // it. + // + // Dispatching the window AFTER this one here too, so the consumer's + // work would overlap the next decode, is a MEASURED DEAD END. It + // doubles how much of the source is decoded at once — peak RSS on a + // 25-row-group scan went 161MB -> 244MB — and returns nothing, + // because the consumer is a blocking operator that eats chunks + // faster than they decode, so there is no consumer work to overlap + // with. Real overlap needs a pipeline that keeps running while the + // scan decodes, which is the rest of Phase 2, not a deeper queue. + // + // `inflight_pending_`, not `batch_`: a one-unit window decodes on + // this thread and never submits, so a `batch_`-based test reports + // "nothing in flight" and drops a window that has already been + // decoded. That silently lost every unit after the first whenever + // the query was not parallel, and every trailing single-unit window + // when it was. + if (!inflight_pending_) { + if (dispatched_ >= units_.size()) { + // A stream must still carry its schema when every unit was + // empty after scan pushdown. Without this carrier, + // MaterializeOperator sees end-of-stream as its first + // result and returns a column-less Table, unlike the + // equivalent eager scan (and unlike a filter above an + // ordinary table source). Keep the first empty unit out + // of the normal pipeline -- several streaming operators + // intentionally do not consume empty chunks -- and emit it + // only when it is the sole result. + if (empty_schema_carrier_.has_value()) { + auto carrier = std::move(*empty_schema_carrier_); + empty_schema_carrier_.reset(); + return std::optional{emit_schema_carrier(std::move(carrier))}; + } + return std::optional{}; + } + dispatch(); + } + harvest(); + } + } + + private: + /// How many units to decode at once. One means serial, which is what a + /// non-parallel query, a single-thread budget, and a call already running + /// inside a pool task all get — the last because submitting from a worker + /// deadlocks against a saturated pool. + static auto unit_window(const ExecutionContext& exec) -> std::size_t { + if (!exec.can_fan_out() || on_worker_pool_thread()) { + return 1; + } + auto& pool = process_worker_pool(); + const std::size_t budget = exec.compute_budget(); + return std::max(1, std::min(budget, pool.size())); + } + + /// Put the next window of units in flight. Returns without blocking: + /// `WorkerPool::submit` is asynchronous, which is what makes the overlap + /// possible — and which is also why everything the body touches is a + /// member rather than a local. + void dispatch() { + window_begin_ = dispatched_; + window_count_ = std::min(window_, units_.size() - dispatched_); + dispatched_ += window_count_; + inflight_.clear(); + inflight_.resize(window_count_); + inflight_pending_ = true; + if (window_count_ == 1) { + // Nothing to overlap and no reason to pay for a pool round trip. + inflight_[0] = decode_unit(0); + return; + } + cursor_.store(0, std::memory_order_relaxed); + batch_ = process_worker_pool().submit(window_count_, [this](std::size_t /*worker*/) { + for (std::size_t i = cursor_.fetch_add(1, std::memory_order_relaxed); i < window_count_; + i = cursor_.fetch_add(1, std::memory_order_relaxed)) { + inflight_[i] = decode_unit(i); + } + }); + } + + /// One unit's decode, as run by a worker. Never throws out of the body: a + /// `Batch` rethrows the first escaped exception at `wait()`, which would + /// lose the other units' errors and unwind through the pool. + auto decode_unit(std::size_t slot) -> std::expected { + try { + auto table = + materialize_deferred_scan_unit(*scan_, plan_, units_[window_begin_ + slot], *exec_); + if (table.has_value()) { + normalize_time_index(*table); + } + return table; + } catch (const std::exception& e) { + return std::unexpected(std::string("streamed scan: ") + e.what()); + } + } + + void harvest() { + if (batch_.has_value()) { + batch_->wait(); + batch_.reset(); + } + ready_ = std::move(inflight_); + inflight_.clear(); + inflight_pending_ = false; + served_ = 0; + } + + /// Turn a decoded unit into the chunk to hand upward, or nullopt when it + /// carries no rows. + /// + /// A unit whose every row the scan's predicates rejected carries nothing. + /// Skipping it is not just an optimization: an empty chunk with columns is + /// a shape some operators would rather not meet, and dropping it changes no + /// result. A column-less chunk is a different thing — it carries a row + /// count for `count()` — and is kept. + auto emit(Table table) -> std::optional { + if (!table.columns.empty() && table.rows() == 0) { + if (!empty_schema_carrier_.has_value()) { + empty_schema_carrier_ = std::move(table); + } + return std::nullopt; + } + // A non-empty result makes a deferred carrier unnecessary. + empty_schema_carrier_.reset(); + unify_categorical_dictionaries(table); + Chunk chunk; + const std::size_t rows = table.rows(); + chunk.set_properties(table.properties()); + chunk.columns = std::move(table.columns); + if (chunk.columns.empty()) { + chunk.logical_rows = table.logical_rows; + } + chunk.sequence = sequence_++; + chunk.row_offset = emitted_rows_; + emitted_rows_ += rows; + return chunk; + } + + /// Materialize the one schema carrier retained when scan pushdown rejected + /// every row. It follows the ordinary source identity convention: first + /// chunk, at row zero. Dictionary unification remains necessary because a + /// zero-row categorical still carries dictionary identity as part of its + /// schema. + auto emit_schema_carrier(Table table) -> Chunk { + unify_categorical_dictionaries(table); + Chunk chunk; + chunk.set_properties(table.properties()); + chunk.columns = std::move(table.columns); + if (chunk.columns.empty()) { + chunk.logical_rows = table.logical_rows; + } + chunk.sequence = sequence_++; + chunk.row_offset = emitted_rows_; + return chunk; + } + + /// Remap every Categorical column onto a dictionary shared by all this + /// source's chunks. + /// + /// Parquet writes one dictionary PER ROW GROUP, and a unit is one row + /// group, so without this each chunk's codes would mean something different + /// from the last one's — and the operators that compare dictionary identity + /// to take a fast path (grouping, joins, the packed key encoder) would be + /// comparing codes across dictionaries that disagree. The whole-file decode + /// never had this problem because it merged the groups' dictionaries + /// itself. `ChunkedParquetSourceOperator` solves it the same way. + /// + /// **One lookup per dictionary ENTRY, never per row.** Interning row by row + /// is a string hash per row, and it does not announce itself: TPC-H's + /// `l_returnflag` and `l_linestatus` are plain `string` in the Arrow schema + /// and only become Categorical because the writer dictionary-encoded them, + /// so a query that never mentions a categorical type still pays. Measured + /// on q01, per-row interning cost 114ms of the scan's 160ms — the entire + /// regression against the materialized path, on the calling thread where + /// nothing could overlap it. A dictionary has a handful of entries and a + /// unit has a million rows; the difference is the whole cost. + void unify_categorical_dictionaries(Table& table) { + using code_type = Column::code_type; + for (std::size_t i = 0; i < table.columns.size(); ++i) { + auto* local = std::get_if>(table.columns[i].column.get()); + if (local == nullptr) { + continue; + } + if (cat_states_.size() <= i) { + cat_states_.resize(table.columns.size()); + } + auto& state = cat_states_[i]; + if (!state.has_value()) { + state.emplace(); + } + if (state->dictionary_ptr() == local->dictionary_ptr()) { + continue; // already speaks the shared dictionary + } + // Intern this chunk's dictionary into the shared one, reading back + // the code each entry landed on. `clear()` drops the codes and + // keeps the dictionary, which is exactly what an accumulator wants. + const auto& dictionary = local->dictionary(); + state->clear(); + for (const auto& value : dictionary) { + state->push_back(value); + } + std::vector remap(dictionary.size()); + for (std::size_t entry = 0; entry < dictionary.size(); ++entry) { + remap[entry] = state->code_at(entry); + } + state->clear(); + + const auto& local_codes = local->codes(); + std::vector codes(local_codes.size()); + for (std::size_t row = 0; row < local_codes.size(); ++row) { + codes[row] = remap[static_cast(local_codes[row])]; + } + table.columns[i].column = std::make_shared( + Column{state->dictionary_ptr(), state->index_ptr(), std::move(codes)}); + } + } + + const DeferredScan* scan_; + DeferredScanPlan plan_; + std::vector units_; + const ExecutionContext* exec_; + std::vector>> cat_states_; + + /// Units decoded and waiting to be served, in unit order, and how many of + /// them have been. + std::vector> ready_; + std::size_t served_ = 0; + /// First empty, column-bearing unit. It becomes a schema carrier only if + /// every unit was empty; otherwise empty units stay invisible to the + /// streaming operators above this source. + std::optional
empty_schema_carrier_; + /// The window currently being decoded. Workers write disjoint slots of + /// this, so it must not be resized while `batch_` is live. + std::vector> inflight_; + std::optional batch_; + /// Whether `inflight_` holds a dispatched window awaiting harvest. Not + /// derivable from `batch_`: a one-unit window is decoded inline. + bool inflight_pending_ = false; + std::atomic cursor_{0}; + std::size_t window_begin_ = 0; + std::size_t window_count_ = 0; + + std::size_t dispatched_ = 0; + std::size_t emitted_rows_ = 0; + std::uint64_t sequence_ = 0; + std::size_t window_ = 1; +}; + +// A one-chunk source owned by one scan-pipeline worker. The worker replaces +// the pending chunk for every source unit it claims, then pulls the private +// row-local chain exactly once. Keeping the chain private is what makes its +// mutable per-operator state safe without locks. +class ScanPipelineSource final : public Operator { + public: + void set(Chunk chunk) { pending_ = std::move(chunk); } + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (!pending_.has_value()) { + return std::optional{}; + } + auto chunk = std::move(*pending_); + pending_.reset(); + return std::optional{std::move(chunk)}; + } + + private: + std::optional pending_; +}; + +struct ScanPipelineWorker { + ScanPipelineSource* source = nullptr; + OperatorPtr chain; +}; + +[[nodiscard]] auto build_scan_pipeline_worker(const std::vector& operators, + const ScalarRegistry* scalars, + const ExternRegistry* externs, + const ExecutionContext& exec) + -> std::expected { + auto source = std::make_unique(); + ScanPipelineWorker worker{.source = source.get(), .chain = std::move(source)}; + for (const MapStep& op_node : operators) { + auto next = build_row_local_map_operator(op_node, std::move(worker.chain), scalars, externs, + exec, true); + if (!next.has_value()) { + return std::unexpected("scan pipeline: " + next.error()); + } + worker.chain = std::move(next.value()); + } + return worker; +} + +/// A bounded source-to-map pipeline. +/// +/// Each worker claims one source unit, decodes it with all globally planned +/// pushdowns intact, immediately runs the row-local operator chain, and +/// publishes the result into a bounded ordered ring. The caller drains that +/// ring into the next blocking operator. Thus decode of unit N+1, row-local +/// work on unit N, and consumption of an earlier unit can all be live at once; +/// there is no materialized table or whole-window wait between those stages. +class PipelinedScanOperator final : public Operator { + public: + PipelinedScanOperator(const DeferredScan& scan, std::vector units, + std::vector workers, const ExecutionContext& exec, + WorkerPool& pool) + : scan_(&scan), + plan_(plan_deferred_scan(scan)), + units_(std::move(units)), + workers_(std::move(workers)), + exec_(&exec), + pool_(&pool), + window_(std::max(workers_.size() * 2, 2)), + ring_(window_, workers_.size()) {} + + ~PipelinedScanOperator() override { cancel_and_join(); } + + PipelinedScanOperator(const PipelinedScanOperator&) = delete; + auto operator=(const PipelinedScanOperator&) -> PipelinedScanOperator& = delete; + PipelinedScanOperator(PipelinedScanOperator&&) = delete; + auto operator=(PipelinedScanOperator&&) -> PipelinedScanOperator& = delete; + + [[nodiscard]] auto next() -> std::expected, std::string> override { + if (finished_) { + return std::optional{}; + } + start(); + + while (next_sequence_ < units_.size()) { + std::optional produced = ring_.take(next_sequence_); + if (!produced.has_value()) { + // Stopped before this unit: cancelled, failed, or out of + // workers. The ring reports the lowest-sequence failure, so the + // message does not depend on which worker lost a race. + if (auto failure = ring_.failure(); failure.has_value()) { + return fail(std::move(*failure)); + } + return fail(interrupt_requested() ? interrupt_message() + : "scan pipeline: missing output unit"); + } + ++next_sequence_; + Chunk chunk = std::move(*produced); + if (!chunk.columns.empty() && chunk.rows() == 0) { + if (!empty_schema_carrier_.has_value()) { + empty_schema_carrier_ = std::move(chunk); + } + continue; + } + + empty_schema_carrier_.reset(); + normalize_categorical_dictionaries(chunk); + restamp(chunk); + return std::optional{std::move(chunk)}; + } + + finish_workers(); + finished_ = true; + if (empty_schema_carrier_.has_value()) { + auto carrier = std::move(*empty_schema_carrier_); + empty_schema_carrier_.reset(); + normalize_categorical_dictionaries(carrier); + restamp(carrier); + return std::optional{std::move(carrier)}; + } + return std::optional{}; + } + + private: + void start() { + if (started_) { + return; + } + started_ = true; + batch_ = pool_->submit(workers_.size(), [this](std::size_t id) { run_worker(id); }); + } + + void run_worker(std::size_t worker_id) noexcept { + // However this worker leaves, it must stop counting as a producer, or + // the consumer waits for a unit that is never coming. + // One-shot scope guard: aggregate-initialised, never copied or moved. + // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions) + struct ExitGuard { + OrderedChunkRing* ring; + ~ExitGuard() { ring->producer_exited(); } + } const guard{&ring_}; + + std::size_t claimed = 0; + try { + auto& worker = workers_[worker_id]; + while (true) { + const std::size_t sequence = cursor_.fetch_add(1, std::memory_order_relaxed); + if (sequence >= units_.size()) { + return; + } + claimed = sequence; + if (ring_.acquire(sequence) == OrderedChunkRing::Acquire::Abandon) { + return; + } + if (interrupt_requested()) { + ring_.cancel(); + return; + } + + auto result = run_unit(worker, sequence); + if (!result.has_value()) { + ring_.record_error(sequence, std::move(result.error())); + return; + } + ring_.publish(sequence, std::move(*result)); + } + } catch (const std::exception& error) { + // Sequence-tagged like any other failure, so the reported message + // obeys the same lowest-sequence rule. Composing it allocates and + // the likeliest exception here is bad_alloc, hence the + // allocation-free fallback underneath. + try { + ring_.record_error(claimed, + "scan pipeline: worker exception: " + std::string(error.what())); + } catch (...) { + ring_.record_fault(claimed, + "scan pipeline: worker exception (no memory to report it)"); + } + } catch (...) { + ring_.record_fault(claimed, "scan pipeline: worker threw a non-standard exception"); + } + } + + [[nodiscard]] auto run_unit(ScanPipelineWorker& worker, std::size_t sequence) + -> std::expected { + auto decoded = materialize_deferred_scan_unit(*scan_, plan_, units_[sequence], *exec_); + if (!decoded.has_value()) { + return std::unexpected(std::move(decoded.error())); + } + normalize_time_index(*decoded); + worker.source->set(table_to_chunk( + std::move(*decoded), + ChunkIdentity{.sequence = sequence, .row_offset = units_[sequence].start})); + auto produced = worker.chain->next(); + if (!produced.has_value()) { + return std::unexpected(std::move(produced.error())); + } + if (!produced->has_value()) { + return std::unexpected("scan pipeline: row-local chain dropped a source unit"); + } + Chunk chunk = std::move(**produced); + if (chunk.sequence != sequence) { + return std::unexpected("scan pipeline: row-local chain reordered a source unit"); + } + return chunk; + } + + void restamp(Chunk& chunk) noexcept { + chunk.sequence = emitted_sequence_++; + chunk.row_offset = emitted_rows_; + emitted_rows_ += chunk.rows(); + } + + // Parquet dictionaries are local to row groups. Ordered publication is the + // one serial point where chunks are remapped onto one shared dictionary, + // preserving the existing streamed-source contract for downstream keys. + void normalize_categorical_dictionaries(Chunk& chunk) { + using code_type = Column::code_type; + if (cat_states_.size() < chunk.columns.size()) { + cat_states_.resize(chunk.columns.size()); + } + for (std::size_t i = 0; i < chunk.columns.size(); ++i) { + auto* local = std::get_if>(chunk.columns[i].column.get()); + if (local == nullptr) { + continue; + } + auto& state = cat_states_[i]; + if (!state.has_value()) { + state.emplace(); + } + if (state->dictionary_ptr() == local->dictionary_ptr()) { + continue; + } + const auto& dictionary = local->dictionary(); + state->clear(); + for (const auto& value : dictionary) { + state->push_back(value); + } + std::vector remap(dictionary.size()); + for (std::size_t entry = 0; entry < dictionary.size(); ++entry) { + remap[entry] = state->code_at(entry); + } + state->clear(); + const auto& local_codes = local->codes(); + std::vector codes(local_codes.size()); + for (std::size_t row = 0; row < local_codes.size(); ++row) { + codes[row] = remap[static_cast(local_codes[row])]; + } + chunk.columns[i].column = std::make_shared( + Column{state->dictionary_ptr(), state->index_ptr(), std::move(codes)}); + } + } + + void finish_workers() { + if (batch_.has_value()) { + batch_->wait(); + batch_.reset(); + } + if (validated_) { + return; + } + validated_ = true; + for (auto& worker : workers_) { + auto trailing = worker.chain->next(); + if (!trailing.has_value()) { + throw std::runtime_error(trailing.error()); + } + if (trailing->has_value()) { + throw std::runtime_error("scan pipeline: unexpected trailing output"); + } + } + } + + void cancel_and_join() noexcept { + ring_.cancel(); + try { + finish_workers(); + } catch (...) { // NOLINT(bugprone-empty-catch) + } + } + + [[nodiscard]] auto fail(std::string message) + -> std::expected, std::string> { + finished_ = true; + cancel_and_join(); + return std::unexpected(std::move(message)); + } + + const DeferredScan* scan_; + DeferredScanPlan plan_; + std::vector units_; + std::vector workers_; + const ExecutionContext* exec_; + WorkerPool* pool_; + std::size_t window_ = 2; + // The same ordered handoff the morsel executor uses: one implementation of + // the bounded, sequence-ordered producer/consumer shape. + OrderedChunkRing ring_; + std::vector>> cat_states_; + std::optional empty_schema_carrier_; + std::optional batch_; + std::atomic cursor_{0}; + std::size_t next_sequence_ = 0; + std::size_t emitted_rows_ = 0; + std::uint64_t emitted_sequence_ = 0; + bool started_ = false; + bool finished_ = false; + bool validated_ = false; +}; + +/// A bounded asynchronous boundary between two pipeline segments. +/// +/// The existing executor is pull-based, which is ideal for operator-local +/// state but normally means a parent cannot start its work until its child has +/// returned from `next()`. This stage retains that contract at both ends while +/// driving its child on a dedicated scheduler thread and holding at most two +/// ordered chunks between them. A breaker below the stage may therefore build +/// or probe the next chunk while a row-local parent (or the next breaker) is +/// working on the preceding one. +/// +/// This deliberately does not borrow a WorkerPool thread. A streamed scan +/// already owns pool tasks, and putting the stage on that same fixed pool +/// reintroduces the saturated-pool deadlock that the scan producer's worker +/// reservation avoids. The query lease limits this to one query, while the +/// builder only inserts stages at breaker boundaries, so this is bounded by +/// plan depth rather than morsel count. +class PipelinedStageOperator final : public Operator { + public: + PipelinedStageOperator(OperatorPtr child, ExecutionProfileEntry* entry) + : child_(std::move(child)), entry_(entry) {} + + ~PipelinedStageOperator() override { cancel_and_join(); } + + PipelinedStageOperator(const PipelinedStageOperator&) = delete; + auto operator=(const PipelinedStageOperator&) -> PipelinedStageOperator& = delete; + PipelinedStageOperator(PipelinedStageOperator&&) = delete; + auto operator=(PipelinedStageOperator&&) -> PipelinedStageOperator& = delete; + + [[nodiscard]] auto next() -> std::expected, std::string> override { + start(); + std::expected, std::string> result = std::optional{}; + { + std::unique_lock lock(mutex_); + { + // Idle, not serial work: waiting on the stage's producer thread. + const RingWaitScope ring_wait; + ready_.wait(lock, [this] { + return !ready_chunks_.empty() || producer_done_ || failure_.has_value(); + }); + } + if (failure_.has_value()) { + result = std::unexpected(std::move(*failure_)); + failure_.reset(); + } else if (!ready_chunks_.empty()) { + result = std::optional{std::move(ready_chunks_.front())}; + ready_chunks_.pop_front(); + } else { + done_ = true; + } + } + space_.notify_one(); + if (!result.has_value()) { + cancel_and_join(); + return result; + } + if (done_) { + join(); + } + return result; + } + + private: + static constexpr std::size_t kCapacity = 2; + + void start() { + if (started_) { + return; + } + started_ = true; + producer_ = std::thread([this] { + // Declare what kind of thread this is. It is deliberately NOT a pool + // worker — it is long-lived and parks on the consumer's ring + // backpressure, which a fixed-size pool cannot host — but it must + // still be countable, and the profiler must not charge its work to + // the calling thread's self time. + const StageThreadScope stage_thread; + produce(); + }); + } + + void produce() noexcept { + try { + while (true) { + { + std::unique_lock lock(mutex_); + // Backpressure: the producer has filled the ring and the + // consumer has not drained it. Idle time on a runtime-owned + // thread, and the mirror image of the consumer's park at the + // other end of the same ring — a large value here means the + // CONSUMER is the bottleneck. `RingWaitScope` routes it to + // this thread's stage ledger. + const RingWaitScope ring_wait; + space_.wait(lock, + [this] { return cancelled_ || ready_chunks_.size() < kCapacity; }); + if (cancelled_) { + return; + } + } + + auto next = [&] { + // Scoped per pull, mirroring ProfiledOperator, so the + // backpressure wait above stays outside it. + const ExecutionProfileScope scope(entry_, ProfilePhase::Next); + return child_->next(); + }(); + if (!next.has_value()) { + fail(std::move(next.error())); + return; + } + if (!next->has_value()) { + { + const std::scoped_lock lock(mutex_); + producer_done_ = true; + } + ready_.notify_all(); + return; + } + + { + const std::scoped_lock lock(mutex_); + if (cancelled_) { + return; + } + // There is one producer. The capacity check immediately + // before `child_->next()` therefore reserves this slot: + // only the consumer can change the queue size meanwhile. + ready_chunks_.push_back(std::move(**next)); + } + ready_.notify_one(); + } + } catch (const std::exception& error) { + fail("pipeline stage: producer exception: " + std::string(error.what())); + } catch (...) { + fail("pipeline stage: producer threw a non-standard exception"); + } + } + + void fail(std::string message) noexcept { + { + const std::scoped_lock lock(mutex_); + if (!failure_.has_value()) { + failure_ = std::move(message); + } + producer_done_ = true; + } + ready_.notify_all(); + } + + void join() noexcept { + if (producer_.joinable()) { + producer_.join(); + } + } + + void cancel_and_join() noexcept { + { + const std::scoped_lock lock(mutex_); + cancelled_ = true; + } + ready_.notify_all(); + space_.notify_all(); + join(); + } + + OperatorPtr child_; + // The operator this stage was built for. `profile_operator` wraps the + // STAGE, not the child, so the producer thread runs unwrapped code and had + // no profile frame at all: anything it submitted to the pool was attributed + // to no operator, and a fully parallel scan read as zero pool work. The + // producer pushes a scope for this entry so its work has an owner. + ExecutionProfileEntry* entry_ = nullptr; + std::thread producer_; + std::deque ready_chunks_; + std::mutex mutex_; + std::condition_variable ready_; + std::condition_variable space_; + std::optional failure_; + bool started_ = false; + bool producer_done_ = false; + bool cancelled_ = false; + bool done_ = false; +}; + +[[nodiscard]] auto make_pipelined_stage(OperatorPtr child, const ExecutionContext& exec, + ExecutionProfileEntry* entry) -> OperatorPtr { + if (!exec.can_fan_out() || on_worker_pool_thread() || process_worker_pool().size() < 2) { + return child; + } + if (exec.parallel_stats != nullptr) { + exec.parallel_stats->pipelined_stages.fetch_add(1, std::memory_order_relaxed); + } + return std::make_unique(std::move(child), entry); +} + +[[nodiscard]] auto make_pipelined_stage_if(OperatorPtr child, bool eligible, + const ExecutionContext& exec, + ExecutionProfileEntry* entry) -> OperatorPtr { + return eligible ? make_pipelined_stage(std::move(child), exec, entry) : std::move(child); +} + +[[nodiscard]] auto scan_pipeline_worker_count(std::size_t unit_count) -> std::size_t { + auto& pool = process_worker_pool(); + if (pool.size() < 2) { + // With one pool thread there is no worker to reserve for a downstream + // operator batch. Running the producer there can deadlock as soon as a + // breaker submits work and waits, so keep the serial window source. + return 0; + } + // The decode pipeline is the one consumer sized against the POOL rather + // than the compute budget: it is what the extra threads were added for. + // `ExecutionContext::parallel_threads` deliberately does not cap it; + // configure_parallel_from_env uses that field for compute only. + const std::size_t budget = pool.size(); + std::size_t workers = std::min({budget, pool.size(), unit_count}); + // A spare thread is only necessary when every pool thread could remain + // parked behind ring backpressure. The ring holds 2W results and workers + // have already claimed at most another W units, so a source of at most 3W + // units necessarily lets one worker exit after the first chunk is released. + // Smaller sources (the common Parquet shape) keep the full decode budget; + // longer sources reserve one thread for downstream batches. + if (workers == pool.size() && unit_count > workers * 3) { + --workers; + } + return workers; +} + +auto make_deferred_scan_source(const DeferredScan& scan, std::vector units, + const ExecutionContext& exec) -> OperatorPtr { + return std::make_unique(scan, std::move(units), exec); +} + +[[nodiscard]] auto build_pipelined_scan(const std::vector& operators, + bool count_as_pipeline, const DeferredScan& scan, + std::vector units, + const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec) + -> std::expected { + const std::size_t worker_count = scan_pipeline_worker_count(units.size()); + if (worker_count == 0) { + return std::unexpected("scan pipeline requires a worker"); + } + std::vector workers; + workers.reserve(worker_count); + for (std::size_t i = 0; i < worker_count; ++i) { + auto worker = build_scan_pipeline_worker(operators, scalars, externs, exec); + if (!worker.has_value()) { + return std::unexpected(std::move(worker.error())); + } + workers.push_back(std::move(*worker)); + } + if (exec.parallel_stats != nullptr) { + if (count_as_pipeline) { + exec.parallel_stats->parallel_pipelines.fetch_add(1, std::memory_order_relaxed); + } + exec.parallel_stats->morsels.fetch_add(units.size(), std::memory_order_relaxed); + exec.parallel_stats->pipelined_scans.fetch_add(1, std::memory_order_relaxed); + } + return std::make_unique(scan, std::move(units), std::move(workers), exec, + process_worker_pool()); +} + +/// A breaker only earns a scheduler thread when its probe input can actually +/// publish more than one source unit. Registered tables and one-unit readers +/// return a single chunk, so staging them merely moves the same serial call to +/// another thread. Keep this structural test at build time: it avoids putting +/// a speculative thread on the hot path and makes the queue capacity an +/// overlap buffer rather than an accidental materialization boundary. +[[nodiscard]] auto has_multi_unit_deferred_scan(const ir::Node& node, const TableRegistry& registry, + const ExecutionContext& exec) -> bool { + if (node.kind() == ir::NodeKind::Scan) { + const auto& scan = ir::node_cast(node); + if (registry.contains(scan.source_name())) { + return false; + } + const auto* deferred = exec.deferred_scan(scan.source_name()); + return deferred != nullptr && deferred->filter == nullptr && + deferred_scan_units(*deferred).size() > 1; + } + return std::ranges::any_of(node.children(), [&](const ir::NodePtr& child) { + return has_multi_unit_deferred_scan(*child, registry, exec); + }); +} + +} // namespace pipeline_executor_detail + +} // namespace ibex::runtime diff --git a/src/runtime/pipeline_executor_internal.hpp b/src/runtime/pipeline_executor_internal.hpp new file mode 100644 index 00000000..f19da37b --- /dev/null +++ b/src/runtime/pipeline_executor_internal.hpp @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (C) 2026 Bob Jansen + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "execution_profile_internal.hpp" +#include "join_chunked_internal.hpp" +#include "physical_plan.hpp" + +namespace ibex::runtime::pipeline_executor_detail { + +[[nodiscard]] auto build_row_local_map_operator(const ir::Node& node, OperatorPtr child, + const ScalarRegistry* scalars, + const ExternRegistry* externs, + const ExecutionContext& exec, + bool preserve_empty_morsels) + -> std::expected; + +[[nodiscard]] auto probe_morsel_workers(const Table& input, const ExecutionContext& exec) + -> std::size_t; +[[nodiscard]] auto build_probe_morsel_pipeline(Table input, const JoinProbeFactory& probe, + std::size_t workers, const ExecutionContext& exec) + -> std::expected; + +[[nodiscard]] auto build_map_pipeline_parallel(const physical::Plan& plan, + const TableRegistry& registry, + const ScalarRegistry* scalars, + const ExternRegistry* externs, + const ExecutionContext& exec, ModelResult* model_out) + -> std::expected; + +[[nodiscard]] auto make_pipelined_stage_if(OperatorPtr child, bool eligible, + const ExecutionContext& exec, + ExecutionProfileEntry* entry) -> OperatorPtr; + +[[nodiscard]] auto scan_pipeline_worker_count(std::size_t unit_count) -> std::size_t; +[[nodiscard]] auto make_deferred_scan_source(const DeferredScan& scan, + std::vector units, + const ExecutionContext& exec) -> OperatorPtr; +[[nodiscard]] auto build_pipelined_scan(const std::vector& operators, + bool count_as_pipeline, const DeferredScan& scan, + std::vector units, + const ScalarRegistry* scalars, + const ExternRegistry* externs, const ExecutionContext& exec) + -> std::expected; + +[[nodiscard]] auto has_multi_unit_deferred_scan(const ir::Node& node, const TableRegistry& registry, + const ExecutionContext& exec) -> bool; + +} // namespace ibex::runtime::pipeline_executor_detail diff --git a/tests/test_interpreter.cpp b/tests/test_interpreter.cpp index 5e8cf86b..8965b63d 100644 --- a/tests/test_interpreter.cpp +++ b/tests/test_interpreter.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -13268,7 +13269,7 @@ TEST_CASE("Streaming first/last match the materializing path (parity)") { // ── Multi-key group-by (the generic grouping path) ──────────────────────────── // // A group-by whose keys are neither all-categorical nor a single string column -// lands in ChunkedAggregateOperator::process_rows_generic, which hashes each +// lands in HashAggregateState::process_rows_generic, which hashes each // row's key columns in place and compares a candidate group's stored key // against the row. Nothing exercised it before: making key comparison always // answer "equal" — which merges unrelated groups — left the whole suite green. @@ -14935,6 +14936,112 @@ TEST_CASE("Pipeline scheduler stages a streamable join before its consumer", CHECK((*w)[3] == 600); } +TEST_CASE("Deferred probe resolving below the stream threshold preserves its drained probe side", + "[runtime][parallel][join][deferred_probe]") { + // Regression for the q18 crash described in kernel-pipeline Phase 4. The + // lazy right starts above kStreamRightThreshold, so the join drains the + // left and publishes a membership filter. That filter leaves one right + // row, putting initialize() back below the threshold. The BuildRight path + // must replay the already-drained left rather than dereference its moved + // operator. + constexpr std::size_t kLeftRows = 20'000; // also clears the probe fan-out floor + constexpr std::size_t kLazyRows = 70'000; // above kStreamRightThreshold (65536) + + runtime::Table left; + left.add_column("k", Column{std::vector(kLeftRows, 7)}); + { + std::vector tags; + tags.reserve(kLeftRows); + for (std::size_t row = 0; row < kLeftRows; ++row) { + tags.push_back(static_cast(row)); + } + left.add_column("tag", Column{std::move(tags)}); + } + runtime::TableRegistry registry; + registry.emplace("build_t", std::move(left)); + auto ir = require_ir("build_t join probe_t on k;"); + + struct RunResult { + runtime::Table table; + std::uint64_t parallel_probes = 0; + bool filter_ready = false; + bool membership_published = false; + }; + const auto run = [&](bool parallel) -> RunResult { + runtime::Table schema; + schema.add_column("k", Column{}); + schema.add_column("payload", Column{}); + auto decode = + [](const std::vector& names, + const runtime::Selection* selection) -> std::expected { + std::vector rows; + if (selection == nullptr) { + rows.resize(kLazyRows); + std::iota(rows.begin(), rows.end(), std::size_t{0}); + } else { + rows.assign(selection->begin(), selection->end()); + } + runtime::Table out; + for (const auto& name : names) { + std::vector values; + values.reserve(rows.size()); + if (name == "k") { + for (const std::size_t row : rows) { + // Exactly one row can match the left side. + values.push_back(row == 0 ? 7 : static_cast(row + 1000)); + } + } else if (name == "payload") { + for (const std::size_t row : rows) { + values.push_back(static_cast(row * 3)); + } + } else { + return std::unexpected("deferred-probe fixture: unknown column " + name); + } + out.add_column(name, Column{std::move(values)}); + } + out.logical_rows = rows.size(); + return out; + }; + + auto slot = std::make_shared(); + runtime::DeferredScanRegistry deferred; + deferred.emplace("probe_t", runtime::DeferredScan{ + .lazy = std::make_shared( + std::move(schema), kLazyRows, decode), + .conjuncts = {}, + .demand = {"k", "payload"}, + .demand_all = false, + .key_column = "k", + .filter = slot, + }); + + runtime::ParallelPipelineStats stats; + runtime::ExecutionContext exec{.deferred_scans = &deferred, .execution_profile = nullptr}; + exec.parallel_threads = parallel ? 4 : 1; + exec.parallel_stats = &stats; + auto out = runtime::interpret(*ir, registry, nullptr, nullptr, nullptr, exec); + REQUIRE(out.has_value()); + return RunResult{ + .table = std::move(*out), + .parallel_probes = stats.parallel_probes.load(), + .filter_ready = slot->ready, + .membership_published = slot->has_membership(), + }; + }; + + auto serial = run(false); + auto parallel = run(true); + CHECK(serial.parallel_probes == 0); + CHECK(parallel.parallel_probes >= 1); + CHECK(serial.filter_ready); + CHECK(parallel.filter_ready); + CHECK(serial.membership_published); + CHECK(parallel.membership_published); + REQUIRE(serial.table.rows() == kLeftRows); + auto mismatch = runtime::compare_tables(serial.table, parallel.table); + CHECK_FALSE(mismatch.has_value()); +} + TEST_CASE("Inner join probe fans out across workers and matches the serial probe", "[runtime][parallel][join]") { // Above the parallel probe's 1<<14 row gate so the fan-out actually diff --git a/tests/test_ir_join_output.cpp b/tests/test_ir_join_output.cpp index 7d0115b1..665bf0ae 100644 --- a/tests/test_ir_join_output.cpp +++ b/tests/test_ir_join_output.cpp @@ -20,6 +20,7 @@ using ibex::ir::JoinKind; using ibex::ir::JoinOutputSide; using ibex::ir::JoinSuffixPolicy; using ibex::ir::plan_join_output; +using ibex::ir::resolve_join_columns; namespace { @@ -65,6 +66,41 @@ TEST_CASE("join output plan: a mapped key keeps both native columns", "[ir][join REQUIRE(out == std::vector{"left_id", "val", "right_id", "other"}); } +TEST_CASE("join column mapping resolves mapped keys once to physical positions", + "[ir][join][schema]") { + const std::vector left{"value", "left_id"}; + const std::vector right{"other", "right_id"}; + const auto left_views = views(left); + const auto right_views = views(right); + const auto mapping = + resolve_join_columns(JoinKind::Inner, {{"left_id", "right_id"}}, left_views, right_views); + REQUIRE(mapping.has_value()); + REQUIRE(mapping->keys.size() == 1); + CHECK(mapping->keys[0].left_index == 1); + CHECK(mapping->keys[0].right_index == 1); + REQUIRE(mapping->output.size() == 4); + CHECK(mapping->output[0].source_index == 0); + CHECK(mapping->output[1].source_index == 1); +} + +TEST_CASE("join column mapping rejects a key absent from either concrete input", + "[ir][join][schema]") { + const std::vector left{"left_id"}; + const std::vector right{"right_id"}; + const auto left_views = views(left); + const auto right_views = views(right); + auto mapping = + resolve_join_columns(JoinKind::Inner, {{"missing", "right_id"}}, left_views, right_views); + REQUIRE_FALSE(mapping.has_value()); + CHECK_THAT(mapping.error(), ContainsSubstring("missing")); + CHECK_THAT(mapping.error(), ContainsSubstring("left input")); + + mapping = + resolve_join_columns(JoinKind::Inner, {{"left_id", "missing"}}, left_views, right_views); + REQUIRE_FALSE(mapping.has_value()); + CHECK_THAT(mapping.error(), ContainsSubstring("right input")); +} + TEST_CASE("join output plan: an explicitly folded mapped key keeps native inputs", "[ir][join][schema]") { const auto plan = plan_of(JoinKind::Inner, {{"id", "right_id", true}}, {"id", "left_value"}, diff --git a/tests/test_ir_schema.cpp b/tests/test_ir_schema.cpp index 130a18b2..036ffa53 100644 --- a/tests/test_ir_schema.cpp +++ b/tests/test_ir_schema.cpp @@ -569,18 +569,28 @@ TEST_CASE("check_joins: widths that share a runtime kind are compatible", "[ir][ REQUIRE_FALSE(ibex::ir::check_joins(join, sources).has_value()); } -TEST_CASE("check_joins: an untyped or open side defers to the runtime", "[ir][schema]") { +TEST_CASE("check_joins: only unknown types or schemas defer to the runtime", "[ir][schema]") { auto join = join_of({{"id", "id"}}); // No type on the right: the column is known to exist, nothing more. auto untyped = two_sources(SchemaInfo::known({{.name = "id", .type = ColumnType::Int64}}), SchemaInfo::known({{.name = "id", .type = std::nullopt}})); REQUIRE_FALSE(ibex::ir::check_joins(join, untyped).has_value()); - // An open schema may carry the key among the columns it does not list. + // Extra physical columns in an open schema are anonymous until named by an + // ascription, so they cannot be used as join keys. auto open = two_sources(SchemaInfo::known({{.name = "id", .type = ColumnType::Int64}}), SchemaInfo::known({{.name = "v", .type = std::nullopt}}, /*open=*/true)); - REQUIRE_FALSE(ibex::ir::check_joins(join, open).has_value()); + auto open_error = ibex::ir::check_joins(join, open); + REQUIRE(open_error.has_value()); + REQUIRE(open_error->find("join key 'id' not found on the right side") != std::string::npos); + + // Naming the key makes it usable while the rest of the physical schema may + // remain open. + auto named_open = two_sources(SchemaInfo::known({{.name = "id", .type = ColumnType::Int64}}), + SchemaInfo::known({{.name = "id", .type = ColumnType::Int64}}, + /*open=*/true)); + REQUIRE_FALSE(ibex::ir::check_joins(join, named_open).has_value()); // An unknown source proves nothing either way. SourceSchemas one{{"left", SchemaInfo::known({{.name = "id", .type = ColumnType::Int64}})}}; diff --git a/tests/test_physical_plan.cpp b/tests/test_physical_plan.cpp index bc2ce729..365bd132 100644 --- a/tests/test_physical_plan.cpp +++ b/tests/test_physical_plan.cpp @@ -22,6 +22,8 @@ #include #include +#include "execution_profile_internal.hpp" +#include "interpreter_internal.hpp" #include "physical_plan.hpp" namespace { @@ -64,6 +66,18 @@ auto serial_plan(const char* source) -> std::pair std::expected { + auto op = runtime::build_operator_from_physical_plan(plan, root, registry, nullptr, nullptr, + exec, nullptr); + if (!op.has_value()) { + return std::unexpected(std::move(op.error())); + } + return runtime::materialize_operator(std::move(*op)); +} + } // namespace TEST_CASE("Physical plan lowers filter+select into a fused map step", "[physical][plan]") { @@ -545,7 +559,8 @@ TEST_CASE("explain_physical renders pipelines and fallback reasons", "[physical] const auto [fallback_tree, fallback] = serial_plan("trades[melt symbol];"); const std::string declined = runtime::physical::explain_physical(fallback); - REQUIRE(declined.find("MaterializedCall(root is not a row-local map)") != std::string::npos); + REQUIRE(declined.find("MaterializedCall(Melt: root is not a row-local map)") != + std::string::npos); // A migrated single-operator breaker renders as a Breaker line, not a // pipeline and not a MaterializedCall. @@ -947,11 +962,11 @@ TEST_CASE("The plan describes the distinct dedup fan-out phase", "[physical][bre REQUIRE(phase.parallelism.worker_cap == 0); } - SECTION("a bare registered scan gets a footer row estimate") { + SECTION("a bare registered scan gets an exact table row estimate") { const auto [tree, plan] = serial_plan("trades[distinct { price }];"); REQUIRE(plan.breaker_phases.size() == 1); const auto& est = plan.breaker_phases.front().parallelism.estimate; - REQUIRE(est.source == runtime::physical::RowEstimate::Source::Footer); + REQUIRE(est.source == runtime::physical::RowEstimate::Source::TableExact); REQUIRE(est.rows == 3); } @@ -1155,61 +1170,154 @@ TEST_CASE("Migrated Tail / TopK / FilterHead / FilterTail execute through the pl } TEST_CASE("The plan describes a streaming join's two fan-out phases", "[physical][breaker]") { - SECTION("hash-build then probe, each with its own floor and strategy") { + SECTION("explicit HashBuild feeds HashProbe with runtime orientation") { const auto [tree, plan] = serial_plan("(a join b on k);"); REQUIRE(plan.migrated); REQUIRE(plan.join.strategy == runtime::physical::JoinStrategy::StreamingProbe); - REQUIRE(plan.breaker_phases.size() == 2); + REQUIRE(plan.streaming_join.has_value()); + REQUIRE(plan.breaker_phases.empty()); - const auto& build = plan.breaker_phases[0]; - REQUIRE(build.name == "hash-build"); - REQUIRE(build.parallelism.strategy == runtime::physical::PartitionStrategy::HeadTable); - REQUIRE(build.parallelism.row_floor == (1U << 17U)); // chunked.cpp build_partitions - REQUIRE(build.parallelism.breaker_max_workers == 64); + const auto& nodes = *plan.streaming_join; + REQUIRE(nodes.build.left_input == plan.join.left_input); + REQUIRE(nodes.build.right_input == plan.join.right_input); + REQUIRE(nodes.probe.left_input == plan.join.left_input); + REQUIRE(nodes.probe.right_input == plan.join.right_input); + REQUIRE(nodes.build.output == runtime::physical::JoinDataKind::RuntimeOrientedBuildOutput); + REQUIRE(nodes.probe.build_input == nodes.build.output); + + const auto& build = nodes.build.parallelism; + REQUIRE(build.strategy == runtime::physical::PartitionStrategy::HeadTable); + REQUIRE(build.row_floor == (1U << 17U)); // chunked.cpp build_partitions + REQUIRE(build.breaker_max_workers == 64); + + const auto& probe = nodes.probe.parallelism; + REQUIRE(probe.strategy == runtime::physical::PartitionStrategy::Range); + REQUIRE(probe.row_floor == (1U << 14U)); // probe_parallel_workers + REQUIRE(probe.breaker_max_workers == 64); + } - const auto& probe = plan.breaker_phases[1]; - REQUIRE(probe.name == "probe"); - REQUIRE(probe.parallelism.strategy == runtime::physical::PartitionStrategy::Range); - REQUIRE(probe.parallelism.row_floor == (1U << 14U)); // probe_parallel_workers - REQUIRE(probe.parallelism.breaker_max_workers == 64); + SECTION("known renamed inputs resolve mapped keys to physical positions") { + runtime::Table left; + left.add_column("value", Column{10, 20}); + left.add_column("id", Column{1, 2}); + runtime::Table right; + right.add_column("other", Column{30, 40}); + right.add_column("id", Column{2, 3}); + runtime::TableRegistry registry; + registry.emplace("left_t", std::move(left)); + registry.emplace("right_t", std::move(right)); + auto tree = require_ir( + "(left_t[rename { left_id = id }] join " + "right_t[rename { right_id = id }] on { left_id = right_id });"); + const auto plan = runtime::physical::plan_physical(*tree, registry, nullptr); + REQUIRE(plan.streaming_join.has_value()); + REQUIRE(plan.streaming_join->columns.has_value()); + REQUIRE(plan.streaming_join->columns->keys.size() == 1); + CHECK(plan.streaming_join->columns->keys[0].left_index == 1); + CHECK(plan.streaming_join->columns->keys[0].right_index == 1); + CHECK(plan.streaming_join->columns->output[1].name == "left_id"); + CHECK(plan.streaming_join->columns->output[3].name == "right_id"); + const std::string text = runtime::physical::explain_physical(plan); + CHECK(text.find("columns: resolved left[1]=right[1] output=4") != std::string::npos); } - SECTION("explain physical renders the strategy line and both phases") { + SECTION("unknown input schemas defer the same mapping to execution") { + const auto [tree, plan] = serial_plan("(a join b on k);"); + REQUIRE(plan.streaming_join.has_value()); + REQUIRE_FALSE(plan.streaming_join->columns.has_value()); + CHECK(runtime::physical::explain_physical(plan).find("columns: deferred") != + std::string::npos); + } + + SECTION("explain physical renders the two nodes and their typed edge") { const auto [tree, plan] = serial_plan("(a join b on k);"); const std::string text = runtime::physical::explain_physical(plan); REQUIRE(text.find("Breaker(Join)") != std::string::npos); REQUIRE(text.find("StreamingProbe") != std::string::npos); - REQUIRE(text.find("hash-build:") != std::string::npos); - REQUIRE(text.find("probe:") != std::string::npos); + REQUIRE(text.find("HashBuild.RuntimeOrientedBuildOutput -> HashProbe.build_input") != + std::string::npos); + REQUIRE(text.find("HashBuild:") != std::string::npos); + REQUIRE(text.find("HashProbe:") != std::string::npos); REQUIRE(text.find("head-table") != std::string::npos); REQUIRE(text.find("MapPipeline") == std::string::npos); } + SECTION("mutating either end of the build output edge is rejected") { + const auto [tree, plan] = serial_plan("(a join b on k);"); + REQUIRE(plan.streaming_join.has_value()); + auto nodes = *plan.streaming_join; + REQUIRE_FALSE(runtime::physical::validate_streaming_join_edge(nodes).has_value()); + + nodes.probe.build_input = runtime::physical::JoinDataKind::None; + auto error = runtime::physical::validate_streaming_join_edge(nodes); + REQUIRE(error.has_value()); + REQUIRE(error->find("does not consume HashBuild") != std::string::npos); + + nodes = *plan.streaming_join; + nodes.probe.right_input = nodes.probe.left_input; + error = runtime::physical::validate_streaming_join_edge(nodes); + REQUIRE(error.has_value()); + REQUIRE(error->find("candidate inputs disagree") != std::string::npos); + } + SECTION("a materializing join carries no fan-out phases") { const auto [tree, plan] = serial_plan("(a left join b on k);"); + REQUIRE_FALSE(plan.streaming_join.has_value()); REQUIRE(plan.breaker_phases.empty()); } + + SECTION("semi join retains its separate streaming operator") { + const auto [tree, plan] = serial_plan("(a semi join b on k);"); + REQUIRE(plan.migrated); + REQUIRE(plan.join.branch == runtime::physical::JoinBranch::SemiAnti); + REQUIRE_FALSE(plan.streaming_join.has_value()); + } } -TEST_CASE("The plan describes a hash aggregate's two fan-out phases", "[physical][breaker]") { - SECTION("partition then finalize, each with its own floor and strategy") { +TEST_CASE("The plan describes a hash aggregate's structural fan-out policies", + "[physical][breaker]") { + SECTION("the hash fallback has four typed structural phases") { + const auto [tree, plan] = + serial_plan("trades[select { total = sum(price) }, by { symbol }];"); + REQUIRE(plan.hash_aggregate.has_value()); + const auto& nodes = *plan.hash_aggregate; + REQUIRE(nodes.discovery.source == tree->children().front().get()); + REQUIRE(nodes.discovery.input == runtime::physical::AggregateDataKind::InputChunks); + REQUIRE(nodes.discovery.output == runtime::physical::AggregateDataKind::DiscoveredGroups); + REQUIRE(nodes.accumulation.input == nodes.discovery.output); + REQUIRE(nodes.accumulation.output == + runtime::physical::AggregateDataKind::AccumulatedGroups); + REQUIRE(nodes.final_ordering.input == nodes.accumulation.output); + REQUIRE(nodes.final_ordering.output == runtime::physical::AggregateDataKind::OrderedGroups); + REQUIRE(nodes.emission.input == nodes.final_ordering.output); + REQUIRE(nodes.emission.output == runtime::physical::AggregateDataKind::OutputChunks); + REQUIRE_FALSE(runtime::physical::validate_hash_aggregate_edges(nodes).has_value()); + } + + SECTION("each structural node owns its fan-out policy") { const auto [tree, plan] = serial_plan("trades[select { total = sum(price) }, by { symbol }];"); REQUIRE(plan.migrated); REQUIRE(plan.aggregate.strategy == runtime::physical::AggregateStrategy::StreamingSorted); - REQUIRE(plan.breaker_phases.size() == 2); - - const auto& partition = plan.breaker_phases[0]; - REQUIRE(partition.name == "partition"); - REQUIRE(partition.parallelism.strategy == runtime::physical::PartitionStrategy::RadixHash); - REQUIRE(partition.parallelism.row_floor == (1U << 18U)); // radix kDefaultPartitionMinRows - REQUIRE(partition.parallelism.breaker_max_workers == 64); - - const auto& finalize = plan.breaker_phases[1]; - REQUIRE(finalize.name == "finalize"); - REQUIRE(finalize.parallelism.strategy == runtime::physical::PartitionStrategy::Owned); - REQUIRE(finalize.parallelism.row_floor == (1U << 17U)); // finalize_owned's merge gate - REQUIRE(finalize.parallelism.breaker_max_workers == 64); + REQUIRE(plan.breaker_phases.empty()); + REQUIRE(plan.hash_aggregate.has_value()); + const auto& nodes = *plan.hash_aggregate; + REQUIRE(nodes.discovery.parallelism.strategy == + runtime::physical::PartitionStrategy::RadixHash); + REQUIRE(nodes.discovery.parallelism.row_floor == (1U << 18U)); + REQUIRE(nodes.discovery.parallelism.breaker_max_workers == 64); + REQUIRE(nodes.accumulation.parallelism.strategy == + runtime::physical::PartitionStrategy::Morsel); + REQUIRE(nodes.accumulation.parallelism.row_floor == (1U << 17U)); + REQUIRE(nodes.accumulation.parallelism.breaker_max_workers == 64); + REQUIRE(nodes.final_ordering.parallelism.strategy == + runtime::physical::PartitionStrategy::Owned); + REQUIRE(nodes.final_ordering.parallelism.row_floor == (1U << 17U)); + REQUIRE(nodes.final_ordering.parallelism.breaker_max_workers == 64); + REQUIRE(nodes.emission.parallelism.strategy == + runtime::physical::PartitionStrategy::Column); + REQUIRE(nodes.emission.parallelism.row_floor == 0); + REQUIRE(nodes.emission.parallelism.breaker_max_workers == 0); } SECTION("explain physical renders the strategy line and both phases") { @@ -1217,12 +1325,54 @@ TEST_CASE("The plan describes a hash aggregate's two fan-out phases", "[physical serial_plan("trades[select { total = sum(price) }, by { symbol }];"); const std::string text = runtime::physical::explain_physical(plan); REQUIRE(text.find("Breaker(Aggregate)") != std::string::npos); - REQUIRE(text.find("partition:") != std::string::npos); - REQUIRE(text.find("finalize:") != std::string::npos); + REQUIRE(text.find("Discovery:") != std::string::npos); + REQUIRE(text.find("Accumulation:") != std::string::npos); + REQUIRE(text.find("FinalOrdering:") != std::string::npos); + REQUIRE(text.find("Emission:") != std::string::npos); + REQUIRE(text.find("Discovery -> Accumulation -> FinalOrdering -> Emission") != + std::string::npos); + REQUIRE( + text.find("InputChunks -> DiscoveredGroups -> AccumulatedGroups -> OrderedGroups -> " + "OutputChunks") != std::string::npos); REQUIRE(text.find("radix-hash") != std::string::npos); REQUIRE(text.find("MapPipeline") == std::string::npos); } + SECTION("a registered scan gives all four nodes its exact input-row bound") { + const auto [tree, plan] = + serial_plan("trades[select { total = sum(price) }, by { symbol }];"); + REQUIRE(plan.hash_aggregate.has_value()); + for (const auto* policy : {&plan.hash_aggregate->discovery.parallelism, + &plan.hash_aggregate->accumulation.parallelism, + &plan.hash_aggregate->final_ordering.parallelism, + &plan.hash_aggregate->emission.parallelism}) { + REQUIRE(policy->estimate.source == runtime::physical::RowEstimate::Source::TableExact); + REQUIRE(policy->estimate.rows == 3); + } + const std::string text = runtime::physical::explain_physical(plan); + REQUIRE(text.find("input estimate 3 rows (table)") != std::string::npos); + } + + SECTION("declared source schemas bind aggregate columns without executing the source") { + auto tree = require_ir("ticks[select { total = sum(volume) }, by { symbol }];"); + const ir::SourceSchemas schemas{{ + "ticks", + ir::SchemaInfo::known({{.name = "timestamp", .type = ir::ColumnType::Timestamp}, + {.name = "symbol", .type = ir::ColumnType::String}, + {.name = "price", .type = ir::ColumnType::Float64}, + {.name = "volume", .type = ir::ColumnType::Int64}}), + }}; + const runtime::TableRegistry empty; + const auto plan = runtime::physical::plan_physical(*tree, empty, nullptr, schemas); + REQUIRE(plan.aggregate.columns.has_value()); + // Required-column projection narrows the physical child to + // [symbol, volume], so the mapping describes that input rather than + // the generator's wider source layout. + REQUIRE(plan.aggregate.columns->group_by == std::vector{0}); + REQUIRE(plan.aggregate.columns->aggregate_inputs == + std::vector>{1}); + } + SECTION("a fused left-join count carries no fan-out phases") { // COUNT(*) grouped by the left key over a left join fuses to one step // that runs whole-table -- no ChunkedAggregateOperator, nothing to fan. @@ -1233,10 +1383,220 @@ TEST_CASE("The plan describes a hash aggregate's two fan-out phases", "[physical "[select { n = count() }, by { o_orderkey }];"); if (plan.aggregate.strategy == runtime::physical::AggregateStrategy::FusedLeftJoinCount) { REQUIRE(plan.breaker_phases.empty()); + REQUIRE_FALSE(plan.hash_aggregate.has_value()); } } } +TEST_CASE("Physical HashBuild and HashProbe consume the resolved join column mapping", + "[physical][breaker][execute][join]") { + runtime::Table left; + left.add_column("value", Column{10, 20, 30}); + left.add_column("id", Column{1, 2, 3}); + runtime::Table right; + right.add_column("other", Column{200, 300, 400}); + right.add_column("id", Column{2, 3, 4}); + runtime::TableRegistry registry; + registry.emplace("left_t", std::move(left)); + registry.emplace("right_t", std::move(right)); + auto tree = require_ir( + "(left_t[rename { left_id = id }] join " + "right_t[rename { right_id = id }] on { left_id = right_id });"); + + runtime::ExecutionContext serial; + serial.parallel_threads = 1; + runtime::ExecutionContext parallel; + parallel.parallel_threads = 4; + const auto s = runtime::interpret(*tree, registry, nullptr, nullptr, nullptr, serial); + const auto p = runtime::interpret(*tree, registry, nullptr, nullptr, nullptr, parallel); + REQUIRE(s.has_value()); + REQUIRE(p.has_value()); + REQUIRE(s->rows() == 2); + REQUIRE(p->rows() == s->rows()); + const auto& left_ids = std::get>(*s->find("left_id")); + const auto& right_ids = std::get>(*s->find("right_id")); + const auto& parallel_left_ids = std::get>(*p->find("left_id")); + const auto& parallel_right_ids = std::get>(*p->find("right_id")); + CHECK(left_ids[0] == 2); + CHECK(left_ids[1] == 3); + CHECK(right_ids[0] == 2); + CHECK(right_ids[1] == 3); + CHECK(parallel_left_ids[0] == left_ids[0]); + CHECK(parallel_left_ids[1] == left_ids[1]); + CHECK(parallel_right_ids[0] == right_ids[0]); + CHECK(parallel_right_ids[1] == right_ids[1]); + + auto plan = runtime::physical::plan_physical(*tree, registry, nullptr); + REQUIRE(plan.streaming_join.has_value()); + REQUIRE(plan.streaming_join->columns.has_value()); + + SECTION("a mutated output position is rejected at the concrete boundary") { + plan.streaming_join->columns->output.back().source_index = 0; + const auto result = execute_physical_plan(plan, *tree, registry, serial); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().find("join column mapping") != std::string::npos); + } + + SECTION("a narrower physical input layout rebinds the complete mapping once") { + auto& columns = *plan.streaming_join->columns; + columns.left_input_names.insert(columns.left_input_names.begin(), "predicate_only"); + for (auto& key : columns.keys) { + ++key.left_index; + } + for (auto& output : columns.output) { + if (output.side == ir::JoinOutputSide::Left) { + ++output.source_index; + } + } + const auto result = execute_physical_plan(plan, *tree, registry, serial); + REQUIRE(result.has_value()); + REQUIRE(result->rows() == s->rows()); + const auto& rebound_left_ids = std::get>(*result->find("left_id")); + const auto& rebound_right_ids = std::get>(*result->find("right_id")); + CHECK(rebound_left_ids[0] == left_ids[0]); + CHECK(rebound_left_ids[1] == left_ids[1]); + CHECK(rebound_right_ids[0] == right_ids[0]); + CHECK(rebound_right_ids[1] == right_ids[1]); + } +} + +TEST_CASE("Physical aggregate consumes its column mapping and rejects mutations", + "[physical][breaker][execute][aggregate]") { + runtime::Table input; + input.add_column("unused", Column{90, 91, 92}); + input.add_column("g", Column{1, 1, 2}); + input.add_column("v", Column{10, 20, 7}); + runtime::TableRegistry registry; + registry.emplace("input", std::move(input)); + auto tree = require_ir("input[select { s = sum(v) }, by { g }];"); + auto plan = runtime::physical::plan_physical(*tree, registry, nullptr); + REQUIRE(plan.aggregate.columns.has_value()); + REQUIRE(plan.aggregate.columns->group_by == std::vector{0}); + REQUIRE(plan.aggregate.columns->aggregate_inputs == std::vector>{1}); + + const auto expected = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE(expected.has_value()); + REQUIRE(expected->rows() == 2); + const auto& expected_s = std::get>(*expected->find("s")); + CHECK(expected_s[0] == 30); + CHECK(expected_s[1] == 7); + + SECTION("a mutated group-key position is rejected at the concrete boundary") { + plan.aggregate.columns->group_by[0] = 1; + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().find("group-by column mapping") != std::string::npos); + } + + SECTION("a mutated aggregate-input position is rejected at the concrete boundary") { + plan.aggregate.columns->aggregate_inputs[0] = 0; + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().find("aggregate input column mapping") != std::string::npos); + } + + SECTION("a deliberately deferred mapping binds once and produces the same result") { + plan.aggregate.columns.reset(); + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE(result.has_value()); + REQUIRE(result->rows() == expected->rows()); + const auto& result_s = std::get>(*result->find("s")); + CHECK(result_s[0] == expected_s[0]); + CHECK(result_s[1] == expected_s[1]); + } + + SECTION("a narrower physical child layout is rebound once at the aggregate boundary") { + // Model a lazy scan that consumed a predicate-only leading column and + // therefore emitted [g, v] although logical inference saw + // [predicate_only, g, v]. The positions are valid for that logical + // schema but stale for the concrete breaker input. + plan.aggregate.columns->input_names = {"predicate_only", "g", "v"}; + plan.aggregate.columns->group_by = {1}; + plan.aggregate.columns->aggregate_inputs = {2}; + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE(result.has_value()); + REQUIRE(result->rows() == expected->rows()); + const auto& result_s = std::get>(*result->find("s")); + CHECK(result_s[0] == expected_s[0]); + CHECK(result_s[1] == expected_s[1]); + } + + SECTION("the serial executor accounts each structural phase independently") { + auto profile = std::make_shared(/*worker_budget=*/1, + /*report=*/false); + auto exec = serial_exec(); + exec.execution_profile = profile; + const auto result = execute_physical_plan(plan, *tree, registry, exec); + REQUIRE(result.has_value()); + + const auto rows = profile->snapshot(); + for (const std::string_view label : {"Aggregate.Discovery", "Aggregate.Accumulation", + "Aggregate.FinalOrdering", "Aggregate.Emission"}) { + const auto phase = + std::ranges::find_if(rows, [&](const auto& row) { return row.label == label; }); + REQUIRE(phase != rows.end()); + CHECK(phase->next_self_ns > 0); + } + } + + SECTION("mutating a structural phase edge is rejected") { + REQUIRE(plan.hash_aggregate.has_value()); + plan.hash_aggregate->final_ordering.input = + runtime::physical::AggregateDataKind::DiscoveredGroups; + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().find("Accumulation -> FinalOrdering") != std::string::npos); + } + + SECTION("removing the structural phase chain is rejected") { + plan.hash_aggregate.reset(); + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().find("no hash-fallback phase chain") != std::string::npos); + } + + SECTION("mutating the discovery input is rejected") { + REQUIRE(plan.hash_aggregate.has_value()); + plan.hash_aggregate->discovery.source = tree.get(); + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE_FALSE(result.has_value()); + CHECK(result.error().find("Discovery input") != std::string::npos); + } +} + +TEST_CASE("Open aggregate schemas defer positions but authorize only declared names", + "[physical][breaker][execute][aggregate]") { + ir::Builder builder; + auto tree = builder.aggregate( + {ir::ColumnRef{.name = "g"}}, + {ir::AggSpec{.func = ir::AggFunc::Sum, .column = {.name = "v"}, .alias = "s"}}); + tree->add_child(builder.scan("open_input")); + + const ir::SourceSchemas open{{ + "open_input", + ir::SchemaInfo::known({{.name = "g", .type = ir::ColumnType::Int64}, + {.name = "v", .type = ir::ColumnType::Int64}}, + /*open=*/true), + }}; + const runtime::TableRegistry none; + const auto plan = runtime::physical::plan_physical(*tree, none, nullptr, open); + REQUIRE(plan.migrated); + REQUIRE_FALSE(plan.aggregate.columns.has_value()); + + runtime::Table input; + input.add_column("extra", Column{4, 5, 6}); + input.add_column("g", Column{1, 1, 2}); + input.add_column("v", Column{10, 20, 7}); + runtime::TableRegistry registry; + registry.emplace("open_input", std::move(input)); + const auto result = execute_physical_plan(plan, *tree, registry, serial_exec()); + REQUIRE(result.has_value()); + REQUIRE(result->rows() == 2); + const auto& sums = std::get>(*result->find("s")); + CHECK(sums[0] == 30); + CHECK(sums[1] == 7); +} + TEST_CASE("The aggregate reads the plan: parallel output equals serial and the fan-out fires", "[physical][breaker][execute]") { // A high-cardinality single-Int64 group-by: past kPairOwnedMinRows so the @@ -1268,6 +1628,9 @@ TEST_CASE("The aggregate reads the plan: parallel output equals serial and the f parallel.parallel_threads = 8; parallel.parallel_min_rows = 0; parallel.parallel_stats = &stats; + auto parallel_profile = + std::make_shared(/*worker_budget=*/8, /*report=*/false); + parallel.execution_profile = parallel_profile; const auto s = runtime::interpret(*ir, registry, nullptr, nullptr, nullptr, serial); const auto p = runtime::interpret(*ir, registry, nullptr, nullptr, nullptr, parallel); @@ -1275,6 +1638,16 @@ TEST_CASE("The aggregate reads the plan: parallel output equals serial and the f REQUIRE(p.has_value()); REQUIRE(stats.parallel_aggregate_partitions.load() > 0); // the radix partition phase fired REQUIRE(stats.parallel_aggregate_finalizes.load() > 0); // and its merge phase + const auto profile_tasks = [](const auto& rows, std::string_view label) { + const auto row = std::ranges::find_if( + rows, [&](const auto& candidate) { return candidate.label == label; }); + REQUIRE(row != rows.end()); + return row->pool_tasks; + }; + const auto parallel_rows = parallel_profile->snapshot(); + REQUIRE(profile_tasks(parallel_rows, "Aggregate.Discovery") > 0); + REQUIRE(profile_tasks(parallel_rows, "Aggregate.FinalOrdering") > 0); + REQUIRE(profile_tasks(parallel_rows, "Aggregate.Emission") > 0); REQUIRE(s->rows() == p->rows()); REQUIRE(s->rows() == static_cast(kKeys)); @@ -1287,6 +1660,101 @@ TEST_CASE("The aggregate reads the plan: parallel output equals serial and the f REQUIRE(sg[i] == pg[i]); REQUIRE(sv[i] == pv[i]); } + + // Execute an explicitly mutated copy through the same physical executor. + // A worker ceiling of one on each structural node must suppress fan-out; + // if the builder recreated factory defaults, these counters would fire. + auto capped_plan = runtime::physical::plan_physical(*ir, registry, nullptr); + REQUIRE(capped_plan.hash_aggregate.has_value()); + capped_plan.hash_aggregate->discovery.parallelism.breaker_max_workers = 1; + capped_plan.hash_aggregate->accumulation.parallelism.breaker_max_workers = 1; + capped_plan.hash_aggregate->final_ordering.parallelism.breaker_max_workers = 1; + capped_plan.hash_aggregate->emission.parallelism.breaker_max_workers = 1; + runtime::ParallelPipelineStats capped_stats; + runtime::ExecutionContext capped_exec; + capped_exec.parallel_threads = 8; + capped_exec.parallel_min_rows = 0; + capped_exec.parallel_stats = &capped_stats; + auto capped_profile = + std::make_shared(/*worker_budget=*/8, /*report=*/false); + capped_exec.execution_profile = capped_profile; + const auto capped = execute_physical_plan(capped_plan, *ir, registry, capped_exec); + REQUIRE(capped.has_value()); + REQUIRE(capped_stats.parallel_aggregate_partitions.load() == 0); + REQUIRE(capped_stats.parallel_aggregate_finalizes.load() == 0); + const auto capped_rows = capped_profile->snapshot(); + for (const std::string_view label : {"Aggregate.Discovery", "Aggregate.Accumulation", + "Aggregate.FinalOrdering", "Aggregate.Emission"}) { + REQUIRE(profile_tasks(capped_rows, label) == 0); + } + REQUIRE(capped->rows() == s->rows()); + const auto& capped_g = std::get>(*capped->find_entry("g")->column); + const auto& capped_v = std::get>(*capped->find_entry("s")->column); + for (std::size_t i = 0; i < s->rows(); ++i) { + REQUIRE(capped_g[i] == sg[i]); + REQUIRE(capped_v[i] == sv[i]); + } +} + +TEST_CASE("Aggregate Accumulation consumes its structural-node worker ceiling", + "[physical][breaker][execute]") { + constexpr std::int64_t kRows = 300'000; + runtime::Table input; + Column groups; + Column values; + groups.reserve(kRows); + values.reserve(kRows); + for (std::int64_t row = 0; row < kRows; ++row) { + groups.push_back(row % 8); + values.push_back(static_cast(row % 97)); + } + input.add_column("g", std::move(groups)); + input.add_column("v", std::move(values)); + runtime::TableRegistry registry; + registry.emplace("many", std::move(input)); + auto tree = require_ir("many[select { s = sum(v), n = count() }, by { g }];"); + + const auto tasks_for = [](const auto& profile, std::string_view label) { + const auto rows = profile->snapshot(); + const auto row = std::ranges::find_if( + rows, [&](const auto& candidate) { return candidate.label == label; }); + REQUIRE(row != rows.end()); + return row->pool_tasks; + }; + const auto run = [&](runtime::physical::Plan plan) { + auto profile = + std::make_shared(/*worker_budget=*/8, /*report=*/false); + runtime::ExecutionContext exec; + exec.parallel_threads = 8; + exec.parallel_min_rows = 0; + exec.execution_profile = profile; + auto result = execute_physical_plan(plan, *tree, registry, exec); + REQUIRE(result.has_value()); + return std::pair{std::move(*result), std::move(profile)}; + }; + + auto baseline_plan = runtime::physical::plan_physical(*tree, registry, nullptr); + auto [baseline, baseline_profile] = run(baseline_plan); + REQUIRE(tasks_for(baseline_profile, "Aggregate.Discovery") > 0); + REQUIRE(tasks_for(baseline_profile, "Aggregate.Accumulation") > 0); + + REQUIRE(baseline_plan.hash_aggregate.has_value()); + baseline_plan.hash_aggregate->accumulation.parallelism.breaker_max_workers = 1; + auto [capped, capped_profile] = run(std::move(baseline_plan)); + REQUIRE(tasks_for(capped_profile, "Aggregate.Discovery") > 0); + REQUIRE(tasks_for(capped_profile, "Aggregate.Accumulation") == 0); + REQUIRE(capped.rows() == baseline.rows()); + const auto& capped_g = std::get>(*capped.find("g")); + const auto& baseline_g = std::get>(*baseline.find("g")); + const auto& capped_s = std::get>(*capped.find("s")); + const auto& baseline_s = std::get>(*baseline.find("s")); + const auto& capped_n = std::get>(*capped.find("n")); + const auto& baseline_n = std::get>(*baseline.find("n")); + for (std::size_t row = 0; row < baseline.rows(); ++row) { + CHECK(capped_g[row] == baseline_g[row]); + CHECK(capped_s[row] == baseline_s[row]); + CHECK(capped_n[row] == baseline_n[row]); + } } TEST_CASE("The join operator reads the hash-build plan: parallel output equals serial",