From 9dd17b1345c23dea3ddf8695ab6e0c8f4a1b52c0 Mon Sep 17 00:00:00 2001 From: Bob Jansen Date: Sat, 29 Aug 2026 18:05:29 +0200 Subject: [PATCH 1/4] data_gen: emit symbol / reference strings as Categorical MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `symbol` in gen_ticks is a handful of distinct values over up to millions of rows — the case dictionary encoding exists for. The row->code vector is exactly the `symbol_idx` the generator already builds, so this is nearly free and also drops the per-row string push. gen_reference's string columns (symbol, name, sector, currency) follow so a join key matches by type and gathering those columns across a join copies codes, not strings. Measured, 10M-row `gen_ticks`, single-threaded native: - group-by by symbol: ~135 ms -> a few ms - (trades join reference on symbol)[agg, by {symbol, name}]: ~950 ms -> ~490 ms (join 566->168 ms, 2-key aggregate 391->22 ms) The residual join cost is unpruned columns — eager pipelines don't get the projection pushdown the lazy/parquet path does (repl.cpp:4819). Separate issue. All 1808 tests pass. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UF1k31UD7JHZujGtgHj57H --- libs/data_gen/data_gen.cpp | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/libs/data_gen/data_gen.cpp b/libs/data_gen/data_gen.cpp index 8f0f5568..9162ab73 100644 --- a/libs/data_gen/data_gen.cpp +++ b/libs/data_gen/data_gen.cpp @@ -102,20 +102,26 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string& } Column ts_col; - Column symbol_col; Column price_col; Column volume_col; ts_col.reserve(rows); - symbol_col.reserve(rows); price_col.reserve(rows); volume_col.reserve(rows); + // `symbol` is a handful of distinct values over up to millions of rows: the + // textbook case for a dictionary-encoded column. Emitting it as Categorical + // (the row->dictionary codes are exactly `symbol_idx`) lets a group-by or a + // join on `symbol` resolve each code once instead of hashing a string per + // row — several times faster on the large tables this generator produces. + using Code = Column::code_type; + std::vector symbol_codes(symbol_idx.begin(), symbol_idx.end()); + Column symbol_col(names, std::move(symbol_codes)); + double price = start_price; auto ts_ms = static_cast(base_ts_ms); for (std::size_t i = 0; i < rows; ++i) { ts_ms += gaps_ms[i]; ts_col.push_back(Timestamp{static_cast(ts_ms * 1'000'000.0)}); - symbol_col.push_back(names[static_cast(symbol_idx[i])]); price += price_steps[i]; price = std::max(price, 0.01); price_col.push_back(price); @@ -213,10 +219,13 @@ auto gen_reference(const std::string& symbols) -> runtime::Table { } } - Column symbol_col; - Column name_col; - Column sector_col; - Column currency_col; + // Dimension-table string columns are Categorical: `symbol` so its type + // matches `gen_ticks`'s join key, and the rest so that gathering them across + // a join to a large fact table copies dictionary codes rather than strings. + Column symbol_col; + Column name_col; + Column sector_col; + Column currency_col; Column lot_size_col; Column tick_size_col; for (const auto& symbol : distinct) { From aa2fe56757b23a46d8fbd19823594ee5bb831bd8 Mon Sep 17 00:00:00 2001 From: Bob Jansen Date: Sat, 29 Aug 2026 18:07:41 +0200 Subject: [PATCH 2/4] data_gen: give each symbol its own price series in gen_ticks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gen_ticks walked a single shared price for every row, so `mean(price)` by symbol converged to the same number — five samples of one trajectory. The doc comment already claimed "random walk per symbol"; now it's true. Each symbol gets a distinct base price (seeded uniform, 0.6x–2.4x start_price) and its own mean-reverting walk. Reversion is needed because a pure additive walk's variance grows with the row count, so at the millions of rows this generator targets the levels would diverge and then read as noise. Example, 50k rows, seed 42: MSFT ~89, NVDA ~130, AAPL ~147, AMZN ~187, GOOG ~229. 1808 tests pass. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UF1k31UD7JHZujGtgHj57H --- libs/data_gen/data_gen.cpp | 19 +++++++++++++++++-- libs/data_gen/data_gen.hpp | 7 +++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/libs/data_gen/data_gen.cpp b/libs/data_gen/data_gen.cpp index 9162ab73..ccb1c06a 100644 --- a/libs/data_gen/data_gen.cpp +++ b/libs/data_gen/data_gen.cpp @@ -89,6 +89,12 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string& if (!price_steps.empty()) { rng.fill_normal(price_steps.data(), rows, 0.0, volatility); } + // Each symbol gets its own base price so a group-by by symbol shows + // distinct levels rather than five samples of one shared walk. + std::vector symbol_base(names.size(), start_price); + if (!symbol_base.empty()) { + rng.fill_uniform(symbol_base.data(), names.size(), start_price * 0.6, start_price * 2.4); + } std::vector volume(rows); if (!volume.empty()) { rng.fill_int(volume.data(), rows, 1, 10'000); @@ -117,13 +123,22 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string& std::vector symbol_codes(symbol_idx.begin(), symbol_idx.end()); Column symbol_col(names, std::move(symbol_codes)); - double price = start_price; + // Per-symbol mean-reverting walk. A pure additive walk's variance grows with + // the row count, so over the millions of rows this generator targets every + // symbol drifts arbitrarily far from its base and the levels reconverge into + // noise. The reversion term (pull toward `symbol_base`) keeps each series + // fluctuating around its own price. + constexpr double kReversion = 0.005; + std::vector symbol_price = symbol_base; auto ts_ms = static_cast(base_ts_ms); for (std::size_t i = 0; i < rows; ++i) { ts_ms += gaps_ms[i]; ts_col.push_back(Timestamp{static_cast(ts_ms * 1'000'000.0)}); - price += price_steps[i]; + const auto sym = static_cast(symbol_idx[i]); + double price = symbol_price[sym] + price_steps[i] + + kReversion * (symbol_base[sym] - symbol_price[sym]); price = std::max(price, 0.01); + symbol_price[sym] = price; price_col.push_back(price); volume_col.push_back(volume[i]); } diff --git a/libs/data_gen/data_gen.hpp b/libs/data_gen/data_gen.hpp index aa00b723..49f1559c 100644 --- a/libs/data_gen/data_gen.hpp +++ b/libs/data_gen/data_gen.hpp @@ -26,8 +26,11 @@ namespace ibex::data_gen { -/// Synthetic tick data: timestamp, symbol, price (random walk per symbol), -/// volume. `symbols` is a comma-separated list, e.g. "AAPL,MSFT,GOOG". +/// Synthetic tick data: timestamp, symbol, price, volume. `symbols` is a +/// comma-separated list, e.g. "AAPL,MSFT,GOOG". Each symbol has its own base +/// price (spread around `start_price`) and its own mean-reverting random walk, +/// so the series stay distinct even over millions of rows. `symbol` is a +/// Categorical column. /// Inter-arrival times are drawn from an Exponential distribution with mean /// `interval_ms` (a Poisson process), not evenly spaced. `start_ts_ms` is the /// first timestamp in Unix milliseconds (0 means "use current wall-clock time"). From a133c9f22e66f5e8889f1bdd20dce77b0705850d Mon Sep 17 00:00:00 2001 From: Bob Jansen Date: Sat, 29 Aug 2026 18:21:59 +0200 Subject: [PATCH 3/4] data_gen: narrow symbol codes explicitly for MSVC The iterator-pair `std::vector(int64_begin, int64_end)` constructor tripped MSVC's C4244 (int64->int32) under /WX. Fill the code vector with an explicit static_cast instead; values are 0..names.size()-1. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UF1k31UD7JHZujGtgHj57H --- libs/data_gen/data_gen.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/data_gen/data_gen.cpp b/libs/data_gen/data_gen.cpp index ccb1c06a..3e50a66e 100644 --- a/libs/data_gen/data_gen.cpp +++ b/libs/data_gen/data_gen.cpp @@ -120,7 +120,10 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string& // join on `symbol` resolve each code once instead of hashing a string per // row — several times faster on the large tables this generator produces. using Code = Column::code_type; - std::vector symbol_codes(symbol_idx.begin(), symbol_idx.end()); + std::vector symbol_codes(rows); + for (std::size_t i = 0; i < rows; ++i) { + symbol_codes[i] = static_cast(symbol_idx[i]); // 0..names.size()-1 + } Column symbol_col(names, std::move(symbol_codes)); // Per-symbol mean-reverting walk. A pure additive walk's variance grows with From 4b9da30abfd918d830df58ff1acf9e7f0419fa58 Mon Sep 17 00:00:00 2001 From: Bob Jansen Date: Sat, 29 Aug 2026 18:25:59 +0200 Subject: [PATCH 4/4] docs: refresh the homepage REPL transcript Recaptured against the current build. The group-by of 10M ticks by symbol dropped from ~101 ms to ~9 ms (Categorical symbol key), generation from ~581 ms to ~261 ms, and the per-symbol prices are now distinct (NVDA ~212, MSFT ~175, AAPL ~80) instead of all landing on ~279. Added a link from the caption to the live browser Playground. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01UF1k31UD7JHZujGtgHj57H --- docs/index.html | 2 +- docs/repl-demo.js | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/index.html b/docs/index.html index 26fc3445..6258803b 100644 --- a/docs/index.html +++ b/docs/index.html @@ -72,7 +72,7 @@

Try the thought, see the table

Sample captured locally using :timing on in the REPL: 10M generated ticks. Timings vary by machine and - run.

+ run. Run Ibex live in your browser →

diff --git a/docs/repl-demo.js b/docs/repl-demo.js index 51975768..aba2f511 100644 --- a/docs/repl-demo.js +++ b/docs/repl-demo.js @@ -4,13 +4,13 @@ const transcript = [ { command: "import data_gen;", - output: "time: 798 us", + output: "time: 1.126 ms", elapsed: 1, }, { command: 'let ticks = gen_ticks(10000000, "AAPL,MSFT,NVDA");', - output: "time: 580.712 ms", - elapsed: 581, + output: "time: 261.4 ms", + elapsed: 261, }, { command: "ticks[select { avg_price = mean(price), traded = sum(volume) }, by symbol, order { avg_price desc }];", @@ -18,12 +18,12 @@ +--------+-----------+-------------+ | symbol | avg_price | traded | +--------+-----------+-------------+ -| "AAPL" | 279.846 | 16671488213 | -| "NVDA" | 279.7652 | 16669109752 | -| "MSFT" | 279.743 | 16662431719 | +| "NVDA" | 212.0345 | 16660628626 | +| "MSFT" | 174.9267 | 16685780924 | +| "AAPL" | 80.42202 | 16657536712 | +--------+-----------+-------------+ -time: 100.744 ms`, - elapsed: 101, +time: 9.3 ms`, + elapsed: 9, }, ];