diff --git a/docs/index.html b/docs/index.html index 26fc3445..6258803b 100644 --- a/docs/index.html +++ b/docs/index.html @@ -72,7 +72,7 @@

Try the thought, see the table

Sample captured locally using :timing on in the REPL: 10M generated ticks. Timings vary by machine and - run.

+ run. Run Ibex live in your browser →

diff --git a/docs/repl-demo.js b/docs/repl-demo.js index 51975768..aba2f511 100644 --- a/docs/repl-demo.js +++ b/docs/repl-demo.js @@ -4,13 +4,13 @@ const transcript = [ { command: "import data_gen;", - output: "time: 798 us", + output: "time: 1.126 ms", elapsed: 1, }, { command: 'let ticks = gen_ticks(10000000, "AAPL,MSFT,NVDA");', - output: "time: 580.712 ms", - elapsed: 581, + output: "time: 261.4 ms", + elapsed: 261, }, { command: "ticks[select { avg_price = mean(price), traded = sum(volume) }, by symbol, order { avg_price desc }];", @@ -18,12 +18,12 @@ +--------+-----------+-------------+ | symbol | avg_price | traded | +--------+-----------+-------------+ -| "AAPL" | 279.846 | 16671488213 | -| "NVDA" | 279.7652 | 16669109752 | -| "MSFT" | 279.743 | 16662431719 | +| "NVDA" | 212.0345 | 16660628626 | +| "MSFT" | 174.9267 | 16685780924 | +| "AAPL" | 80.42202 | 16657536712 | +--------+-----------+-------------+ -time: 100.744 ms`, - elapsed: 101, +time: 9.3 ms`, + elapsed: 9, }, ]; diff --git a/libs/data_gen/data_gen.cpp b/libs/data_gen/data_gen.cpp index 8f0f5568..3e50a66e 100644 --- a/libs/data_gen/data_gen.cpp +++ b/libs/data_gen/data_gen.cpp @@ -89,6 +89,12 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string& if (!price_steps.empty()) { rng.fill_normal(price_steps.data(), rows, 0.0, volatility); } + // Each symbol gets its own base price so a group-by by symbol shows + // distinct levels rather than five samples of one shared walk. + std::vector symbol_base(names.size(), start_price); + if (!symbol_base.empty()) { + rng.fill_uniform(symbol_base.data(), names.size(), start_price * 0.6, start_price * 2.4); + } std::vector volume(rows); if (!volume.empty()) { rng.fill_int(volume.data(), rows, 1, 10'000); @@ -102,22 +108,40 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string& } Column ts_col; - Column symbol_col; Column price_col; Column volume_col; ts_col.reserve(rows); - symbol_col.reserve(rows); price_col.reserve(rows); volume_col.reserve(rows); - double price = start_price; + // `symbol` is a handful of distinct values over up to millions of rows: the + // textbook case for a dictionary-encoded column. Emitting it as Categorical + // (the row->dictionary codes are exactly `symbol_idx`) lets a group-by or a + // join on `symbol` resolve each code once instead of hashing a string per + // row — several times faster on the large tables this generator produces. + using Code = Column::code_type; + std::vector symbol_codes(rows); + for (std::size_t i = 0; i < rows; ++i) { + symbol_codes[i] = static_cast(symbol_idx[i]); // 0..names.size()-1 + } + Column symbol_col(names, std::move(symbol_codes)); + + // Per-symbol mean-reverting walk. A pure additive walk's variance grows with + // the row count, so over the millions of rows this generator targets every + // symbol drifts arbitrarily far from its base and the levels reconverge into + // noise. The reversion term (pull toward `symbol_base`) keeps each series + // fluctuating around its own price. + constexpr double kReversion = 0.005; + std::vector symbol_price = symbol_base; auto ts_ms = static_cast(base_ts_ms); for (std::size_t i = 0; i < rows; ++i) { ts_ms += gaps_ms[i]; ts_col.push_back(Timestamp{static_cast(ts_ms * 1'000'000.0)}); - symbol_col.push_back(names[static_cast(symbol_idx[i])]); - price += price_steps[i]; + const auto sym = static_cast(symbol_idx[i]); + double price = symbol_price[sym] + price_steps[i] + + kReversion * (symbol_base[sym] - symbol_price[sym]); price = std::max(price, 0.01); + symbol_price[sym] = price; price_col.push_back(price); volume_col.push_back(volume[i]); } @@ -213,10 +237,13 @@ auto gen_reference(const std::string& symbols) -> runtime::Table { } } - Column symbol_col; - Column name_col; - Column sector_col; - Column currency_col; + // Dimension-table string columns are Categorical: `symbol` so its type + // matches `gen_ticks`'s join key, and the rest so that gathering them across + // a join to a large fact table copies dictionary codes rather than strings. + Column symbol_col; + Column name_col; + Column sector_col; + Column currency_col; Column lot_size_col; Column tick_size_col; for (const auto& symbol : distinct) { diff --git a/libs/data_gen/data_gen.hpp b/libs/data_gen/data_gen.hpp index aa00b723..49f1559c 100644 --- a/libs/data_gen/data_gen.hpp +++ b/libs/data_gen/data_gen.hpp @@ -26,8 +26,11 @@ namespace ibex::data_gen { -/// Synthetic tick data: timestamp, symbol, price (random walk per symbol), -/// volume. `symbols` is a comma-separated list, e.g. "AAPL,MSFT,GOOG". +/// Synthetic tick data: timestamp, symbol, price, volume. `symbols` is a +/// comma-separated list, e.g. "AAPL,MSFT,GOOG". Each symbol has its own base +/// price (spread around `start_price`) and its own mean-reverting random walk, +/// so the series stay distinct even over millions of rows. `symbol` is a +/// Categorical column. /// Inter-arrival times are drawn from an Exponential distribution with mean /// `interval_ms` (a Poisson process), not evenly spaced. `start_ts_ms` is the /// first timestamp in Unix milliseconds (0 means "use current wall-clock time").