Webui - further improvements

docs/index.html

@@ -72,7 +72,7 @@ <h2 id="repl-demo-title">Try the thought, see the table</h2>
       </p>
       <p class="repl-caption">Sample captured locally using <code>:timing
         on</code> in the REPL: 10M generated ticks. Timings vary by machine and
-      run.</p>
+      run. <a href="./playground.html">Run Ibex live in your browser &#8594;</a></p>
     </div>
     <div class="repl-terminal" data-repl-demo aria-label="Animated Ibex REPL session">
       <div class="terminal-titlebar">

docs/repl-demo.js

@@ -4,26 +4,26 @@
   const transcript = [
     {
       command: "import data_gen;",
-      output: "time: 798 us",
+      output: "time: 1.126 ms",
       elapsed: 1,
     },
     {
       command: 'let ticks = gen_ticks(10000000, "AAPL,MSFT,NVDA");',
-      output: "time: 580.712 ms",
-      elapsed: 581,
+      output: "time: 261.4 ms",
+      elapsed: 261,
     },
     {
       command: "ticks[select { avg_price = mean(price), traded = sum(volume) }, by symbol, order { avg_price desc }];",
       output: `rows: 3
 +--------+-----------+-------------+
 | symbol | avg_price | traded      |
 +--------+-----------+-------------+
-| "AAPL" | 279.846   | 16671488213 |
-| "NVDA" | 279.7652  | 16669109752 |
-| "MSFT" | 279.743   | 16662431719 |
+| "NVDA" | 212.0345  | 16660628626 |
+| "MSFT" | 174.9267  | 16685780924 |
+| "AAPL" | 80.42202  | 16657536712 |
 +--------+-----------+-------------+
-time: 100.744 ms`,
-      elapsed: 101,
+time: 9.3 ms`,
+      elapsed: 9,
     },
   ];
 

libs/data_gen/data_gen.cpp

@@ -89,6 +89,12 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string&
     if (!price_steps.empty()) {
         rng.fill_normal(price_steps.data(), rows, 0.0, volatility);
     }
+    // Each symbol gets its own base price so a group-by by symbol shows
+    // distinct levels rather than five samples of one shared walk.
+    std::vector<double> symbol_base(names.size(), start_price);
+    if (!symbol_base.empty()) {
+        rng.fill_uniform(symbol_base.data(), names.size(), start_price * 0.6, start_price * 2.4);
+    }
     std::vector<std::int64_t> volume(rows);
     if (!volume.empty()) {
         rng.fill_int(volume.data(), rows, 1, 10'000);
@@ -102,22 +108,40 @@ auto gen_ticks(const runtime::RngBridge& rng, std::int64_t n, const std::string&
     }
 
     Column<Timestamp> ts_col;
-    Column<std::string> symbol_col;
     Column<double> price_col;
     Column<std::int64_t> volume_col;
     ts_col.reserve(rows);
-    symbol_col.reserve(rows);
     price_col.reserve(rows);
     volume_col.reserve(rows);
 
-    double price = start_price;
+    // `symbol` is a handful of distinct values over up to millions of rows: the
+    // textbook case for a dictionary-encoded column. Emitting it as Categorical
+    // (the row->dictionary codes are exactly `symbol_idx`) lets a group-by or a
+    // join on `symbol` resolve each code once instead of hashing a string per
+    // row — several times faster on the large tables this generator produces.
+    using Code = Column<Categorical>::code_type;
+    std::vector<Code> symbol_codes(rows);
+    for (std::size_t i = 0; i < rows; ++i) {
+        symbol_codes[i] = static_cast<Code>(symbol_idx[i]);  // 0..names.size()-1
+    }
+    Column<Categorical> symbol_col(names, std::move(symbol_codes));
+
+    // Per-symbol mean-reverting walk. A pure additive walk's variance grows with
+    // the row count, so over the millions of rows this generator targets every
+    // symbol drifts arbitrarily far from its base and the levels reconverge into
+    // noise. The reversion term (pull toward `symbol_base`) keeps each series
+    // fluctuating around its own price.
+    constexpr double kReversion = 0.005;
+    std::vector<double> symbol_price = symbol_base;
     auto ts_ms = static_cast<double>(base_ts_ms);
     for (std::size_t i = 0; i < rows; ++i) {
         ts_ms += gaps_ms[i];
         ts_col.push_back(Timestamp{static_cast<std::int64_t>(ts_ms * 1'000'000.0)});
-        symbol_col.push_back(names[static_cast<std::size_t>(symbol_idx[i])]);
-        price += price_steps[i];
+        const auto sym = static_cast<std::size_t>(symbol_idx[i]);
+        double price = symbol_price[sym] + price_steps[i] +
+                       kReversion * (symbol_base[sym] - symbol_price[sym]);
         price = std::max(price, 0.01);
+        symbol_price[sym] = price;
         price_col.push_back(price);
         volume_col.push_back(volume[i]);
     }
@@ -213,10 +237,13 @@ auto gen_reference(const std::string& symbols) -> runtime::Table {
         }
     }
 
-    Column<std::string> symbol_col;
-    Column<std::string> name_col;
-    Column<std::string> sector_col;
-    Column<std::string> currency_col;
+    // Dimension-table string columns are Categorical: `symbol` so its type
+    // matches `gen_ticks`'s join key, and the rest so that gathering them across
+    // a join to a large fact table copies dictionary codes rather than strings.
+    Column<Categorical> symbol_col;
+    Column<Categorical> name_col;
+    Column<Categorical> sector_col;
+    Column<Categorical> currency_col;
     Column<std::int64_t> lot_size_col;
     Column<double> tick_size_col;
     for (const auto& symbol : distinct) {

libs/data_gen/data_gen.hpp

@@ -26,8 +26,11 @@
 
 namespace ibex::data_gen {
 
-/// Synthetic tick data: timestamp, symbol, price (random walk per symbol),
-/// volume. `symbols` is a comma-separated list, e.g. "AAPL,MSFT,GOOG".
+/// Synthetic tick data: timestamp, symbol, price, volume. `symbols` is a
+/// comma-separated list, e.g. "AAPL,MSFT,GOOG". Each symbol has its own base
+/// price (spread around `start_price`) and its own mean-reverting random walk,
+/// so the series stay distinct even over millions of rows. `symbol` is a
+/// Categorical column.
 /// Inter-arrival times are drawn from an Exponential distribution with mean
 /// `interval_ms` (a Poisson process), not evenly spaced. `start_ts_ms` is the
 /// first timestamp in Unix milliseconds (0 means "use current wall-clock time").