diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a3cef7..573f4ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,7 +87,7 @@ jobs: fi - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: tag_name: ${{ github.ref_name }} name: ${{ github.ref_name }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c75541..6e863cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [0.3.1] — 2026-09-14 + +### Added +- Server-scale benchmark profile (#52): seeded 100K-vector dataset generator (`scripts/gen_dataset.py`) and a `server_scale` harness measuring plain / `working_dim=256` / cascade modes against exact f32 ground truth, with results, a 2K/10K/100K scale curve, and positioning in the new "Server scale" section of `docs/BENCHMARK.md`. Headline: compression holds at 4.78x/100K; recall is N-dependent (5-bit r@10 0.974 → 0.850) because true-neighbor margins collapse as N grows — vecq's measured sweet spot is the local/on-device profile up to ~10K vectors (r@10 0.932 @ 18 ms/q single-thread); the 2-bit cascade is not a single-threaded throughput win at server N. + +### Changed +- Crate metadata honesty pass (#49): new `vecq-core` description — "Training-free vector quantization (4/5/6-bit) and search — the SQLite profile for edge vector storage" — plus `homepage`, `keywords`, and `categories`, so the crates.io page reflects the current scope; README gains badges and benchmark-reproduce folds. +- Dependency/CI bumps: `softprops/action-gh-release` 2 → 3 (#48), usearch 2.26.1 → 2.26.2 in the bench harness (#50). No library code changes since 0.3.0. + ## [0.3.0] — 2026-08-30 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc9e715..c6d7c84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,3 +32,27 @@ This document helps you decide *whether* and *how* to contribute in a way that's ## Security See [SECURITY.md](SECURITY.md). + +## CLA + +All contributions (code, docs, tests, configuration) require a signed +Contributor License Agreement before a pull request can be merged: + +- 📋 **Individual?** → [Sign the Individual CLA](https://codecoradev.github.io/cla/?type=individual) +- 🏢 **Contributing on behalf of a company?** → [Sign the Corporate CLA](https://codecoradev.github.io/cla/?type=corporate) + +The CLA is a license agreement, not a copyright assignment — you keep +ownership of your work. Signing takes a couple of minutes and is stored +in the [codecoradev/.github](https://github.com/codecoradev/.github) +repository; a bot checks it automatically on every pull request. + +## Contributions are unpaid + +Contributing to this project is **voluntary and unpaid**. There is no +compensation, payment, bounty, or financial reward of any kind for +contributions — now or in the future. You contribute on your own time, +at your own discretion, because you want to improve the project. + +If any paid-contribution program is ever introduced, it will be announced +explicitly and this document will be updated. Until then, assume every +contribution is volunteer work under the license terms above. diff --git a/Cargo.lock b/Cargo.lock index 0a2da45..b08bf12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2193,9 +2193,9 @@ dependencies = [ [[package]] name = "usearch" -version = "2.26.1" +version = "2.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884002b91c545f43b67e9eadbef70834502ad14c32df1ce8d72bdb56ab7d8be3" +checksum = "b43f3a7c8d3b100d53c1a34e9ce4cd76cc83627bdb4fcd035d4e82102778aeee" dependencies = [ "cxx", "cxx-build", @@ -2216,7 +2216,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vecq-bench" -version = "0.3.0" +version = "0.3.1" dependencies = [ "memmap2", "rabitq-rs", @@ -2227,7 +2227,7 @@ dependencies = [ [[package]] name = "vecq-core" -version = "0.3.0" +version = "0.3.1" dependencies = [ "rand 0.10.2", ] diff --git a/Cargo.toml b/Cargo.toml index d790114..daa1c7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,12 +3,12 @@ resolver = "2" members = ["crates/vecq-core", "crates/vecq-bench"] [workspace.package] -version = "0.3.0" +version = "0.3.1" edition = "2021" license = "Apache-2.0" repository = "https://github.com/codecoradev/vecq" homepage = "https://github.com/codecoradev/vecq" -keywords = ["vector", "quantization", "search", "embedded", "no-std"] +keywords = ["vector", "quantization", "search", "embedded", "mobile"] categories = ["algorithms", "data-structures", "science"] authors = ["CodeCora "] diff --git a/README.md b/README.md index a86da15..c5e8f8a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # vecq +[![CI](https://github.com/codecoradev/vecq/actions/workflows/ci.yml/badge.svg)](https://github.com/codecoradev/vecq/actions/workflows/ci.yml) +[![Crates.io](https://img.shields.io/crates/v/vecq-core.svg)](https://crates.io/crates/vecq-core) +[![docs.rs](https://img.shields.io/docsrs/vecq-core)](https://docs.rs/vecq-core) +[![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) + > Training-free vector quantization at configurable width (4/5/6-bit) and search — the "SQLite profile" for vector storage on edge devices. `vecq` compresses dense embeddings **~5x** into a single deterministic file, using a zero-dependency pure-Rust crate. No training pass, no server, no C++. @@ -13,6 +18,21 @@ vecq residual: 1,028 bytes/vector (3.0x smaller, recall@10 0.984) It is the semantic-search engine for on-device and offline-first workloads — the layer below [uteke](https://github.com/codecoradev/uteke), the SQLite-based memory engine, where it is available as an optional search backend. +## Installation + +```sh +cargo add vecq-core +``` + +or in `Cargo.toml`: + +```toml +[dependencies] +vecq-core = "0.3" +``` + +The core crate is dependency-free — adding it pulls nothing beyond `std`. The benchmarking harness used for every number in this README lives in [`crates/vecq-bench`](crates/vecq-bench). + ## Why vecq | | vecq | HNSW libraries (usearch etc.) | server engines (Qdrant) | @@ -46,6 +66,17 @@ Based on techniques validated in the RaBitQ / MonaVec line of research (random r Real EmbeddingGemma, 768-dim, aarch64 release, n=2,000 — full methodology and the width matrix in [`docs/BENCHMARK.md`](docs/BENCHMARK.md). +
+Reproduce + +```sh +cargo run --release -p vecq-bench --bin widths # full 4/5/6-bit + residual matrix +cargo run --release -p vecq-bench --bin real # recall/latency vs exact cosine ground truth +``` + +The dataset is 2,000 base vectors + 100 queries (768-dim) embedded with EmbeddingGemma 300M (Q4 ONNX) over a synthetic corpus — setup described in [`docs/BENCHMARK.md`](docs/BENCHMARK.md). Recall values are deterministic across runs and must match the tables bit-for-bit; latencies are aarch64-host numbers (ordering replicates everywhere, absolute times vary by machine). +
+ ## Usage ```rust @@ -115,6 +146,17 @@ let hits = index.search(&query, 10); Default width (5-bit), aarch64, single-threaded, 2,000 real EmbeddingGemma vectors (768-dim): search **3.21 ms/query**, build **75 ms**, recall@10 **0.979**. The 4-bit width trades to 0.89 ms/query @ 0.958; residual trades up to 0.984 @ 1.76 ms/query. Full methodology, the width matrix, the usearch comparison, and the per-architecture scoring-path matrix (NEON / AVX2 / scalar, all bit-identical) in [`docs/BENCHMARK.md`](docs/BENCHMARK.md). +
+Reproduce + +```sh +cargo run --release -p vecq-bench --bin real # recall + latency vs exact f32 cosine ground truth +cargo run --release -p vecq-bench --bin vs_usearch # head-to-head against usearch HNSW (f32) +``` + +Ground truth is exact brute-force cosine on the same queries. Recall is deterministic across runs; timings vary by host — the scoring-path matrix in [`docs/BENCHMARK.md`](docs/BENCHMARK.md) lists the expected per-architecture behavior. +
+ ## Persistence & serving - **SQLite BLOB** for mutable, transactional, embedded storage: schema, save/load pattern, atomicity, measured latencies at 1k/10k/50k vectors, and pitfalls — [`docs/SQLITE.md`](docs/SQLITE.md). diff --git a/crates/vecq-bench/src/bin/scale_probe.rs b/crates/vecq-bench/src/bin/scale_probe.rs new file mode 100644 index 0000000..0f20b05 --- /dev/null +++ b/crates/vecq-bench/src/bin/scale_probe.rs @@ -0,0 +1,78 @@ +//! Diagnostic: does in-memory scoring (f32 scales) differ from the persisted +//! artifact (f16 scales, reloaded from to_bytes)? +use std::fs; + +use vecq_core::VecqIndex; + +fn load_f32(path: &str, n: usize, dim: usize) -> Vec> { + let bytes = fs::read(path).expect("read file"); + (0..n) + .map(|i| { + (0..dim) + .map(|j| { + f32::from_le_bytes( + bytes[(i * dim + j) * 4..(i * dim + j) * 4 + 4] + .try_into() + .unwrap(), + ) + }) + .collect() + }) + .collect() +} + +fn meta_get(meta: &str, k: &str) -> usize { + let i = meta.find(&format!("\"{k}\"")).expect(k) + k.len() + 4; + let rest = &meta[i..]; + let end = rest.find(|c: char| !c.is_ascii_digit()).unwrap(); + rest[..end].parse().unwrap() +} + +fn main() { + let dir = "/tmp/vecq-bench"; + let meta = fs::read_to_string(format!("{dir}/meta.json")).unwrap(); + let (nb, nq, dim) = ( + meta_get(&meta, "n_base"), + meta_get(&meta, "n_query"), + meta_get(&meta, "dim"), + ); + let base = load_f32(&format!("{dir}/base.f32"), nb, dim); + let queries = load_f32(&format!("{dir}/queries.f32"), nq, dim); + + for &bits in &[5u8, 4] { + let mut idx = VecqIndex::new(dim, 42); + if bits != 5 { + idx.set_bits(bits); + } + for v in &base { + idx.add(v); + } + let bytes = idx.to_bytes(); + let reloaded = VecqIndex::from_bytes(&bytes).expect("reload"); + + let mut score_diffs = 0usize; + let mut max_delta = 0.0f32; + let mut list_diffs = 0usize; + for q in &queries { + let a = idx.search(q, 10); + let b = reloaded.search(q, 10); + if a != b { + list_diffs += 1; + } + // compare scores for the ids returned by the in-memory search + for (id, sa) in &a { + let sb = b.iter().find(|(i, _)| i == id).map(|(_, s)| *s); + if let Some(sb) = sb { + let d = (sa - sb).abs(); + if d > 0.0 { + score_diffs += 1; + max_delta = max_delta.max(d); + } + } + } + } + println!( + "bits={bits}: top10 list diffs in-memory vs reloaded = {list_diffs}/{nq}, nonzero score deltas = {score_diffs}, max |delta| = {max_delta:.2e}" + ); + } +} diff --git a/crates/vecq-bench/src/bin/server_scale.rs b/crates/vecq-bench/src/bin/server_scale.rs new file mode 100644 index 0000000..a19c853 --- /dev/null +++ b/crates/vecq-bench/src/bin/server_scale.rs @@ -0,0 +1,323 @@ +//! Server-scale benchmark profile for #52: 100K real EmbeddingGemma vectors +//! (768-dim, seeded), measuring plain / working_dim-256 modes through a +//! zero-copy `VecqView` over mmap, plus the 4-bit cascade (#22) rescore on +//! the full index, against exact f32 cosine ground truth — the same +//! methodology as the `real` harness, so the server and edge tables share +//! one ground-truth pipeline. +//! +//! Generate the dataset first (see docs/BENCHMARK.md "Server scale"): +//! python3 scripts/gen_dataset.py --n-base 100000 --n-query 200 \ +//! --out /tmp/vecq-bench-100k +//! cargo run --release -p vecq-bench --bin server_scale +//! +//! Options: `--dir ` overrides the dataset dir, `--skip-gt` skips the +//! exact ground-truth pass (timing-only rerun; recall columns are omitted). +//! +//! Everything here is single-threaded by design — the parallel scan path +//! (#51) will extend this harness with a threads dimension. Recall values +//! are deterministic for a given dataset; timings are host-specific. + +use std::fs; +use std::io::Write; +use std::time::{Duration, Instant}; + +use memmap2::Mmap; +use vecq_core::{VecqIndex, VecqView}; + +fn load_f32(path: &str, n: usize, dim: usize) -> Vec> { + let bytes = fs::read(path).expect("read file"); + assert_eq!(bytes.len(), n * dim * 4, "file size mismatch"); + (0..n) + .map(|i| { + (0..dim) + .map(|j| { + f32::from_le_bytes( + bytes[(i * dim + j) * 4..(i * dim + j) * 4 + 4] + .try_into() + .unwrap(), + ) + }) + .collect() + }) + .collect() +} + +fn cosine(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum() +} + +/// One result row of the final table. +struct Row { + mode: String, + build: Option, + file_bytes: usize, + ms_q: f64, + recall1: Option, + recall10: Option, + note: &'static str, +} + +fn meta_get(meta: &str, k: &str) -> usize { + let i = meta.find(&format!("\"{k}\"")).expect(k) + k.len() + 4; + let rest = &meta[i..]; + let end = rest.find(|c: char| !c.is_ascii_digit()).unwrap(); + rest[..end].parse().unwrap() +} + +/// Build an index over `base` with the given width / working dim. +fn build_index( + dim: usize, + working_dim: usize, + bits: u8, + base: &[Vec], +) -> (VecqIndex, Duration) { + let t0 = Instant::now(); + let mut idx = if working_dim == dim { + VecqIndex::new(dim, 42) + } else { + VecqIndex::with_working_dim(dim, working_dim, 42) + }; + if bits != 5 { + idx.set_bits(bits); + } + for v in base { + idx.add(v); + } + (idx, t0.elapsed()) +} + +fn write_index(dir: &str, name: &str, idx: &VecqIndex) -> (String, usize) { + let bytes = idx.to_bytes(); + let path = format!("{dir}/{name}.vecq"); + let mut f = fs::File::create(&path).expect("create file"); + f.write_all(&bytes).expect("write file"); + (path, bytes.len()) +} + +/// Time `queries` against an mmap'd view, scoring recall against `gt` +/// (exact top-10 index lists, `None` with --skip-gt). +fn bench_view( + path: &str, + queries: &[Vec], + gt: Option<&[Vec]>, +) -> (f64, Option, Option) { + let file = fs::File::open(path).expect("open index file"); + let map = unsafe { Mmap::map(&file).expect("mmap") }; + let view = VecqView::from_bytes(&map).expect("parse view"); + let mut hits1 = 0usize; + let mut hits10 = 0usize; + let t = Instant::now(); + for (qi, q) in queries.iter().enumerate() { + let res = view.search(q, 10); + if let Some(gt) = gt { + if res[0].0 == gt[qi][0] { + hits1 += 1; + } + for t in >[qi] { + if res.iter().any(|(i, _)| i == t) { + hits10 += 1; + } + } + } + } + let elapsed = t.elapsed(); + let nq = queries.len() as f64; + let ms_q = elapsed.as_secs_f64() * 1e3 / nq; + let recall1 = gt.map(|_| hits1 as f32 / queries.len() as f32); + let recall10 = gt.map(|_| hits10 as f32 / (queries.len() * 10) as f32); + (ms_q, recall1, recall10) +} + +fn main() { + let mut dir = "/tmp/vecq-bench-100k".to_string(); + let mut skip_gt = false; + let mut args = std::env::args().skip(1); + while let Some(a) = args.next() { + match a.as_str() { + "--dir" => dir = args.next().expect("--dir needs a value"), + "--skip-gt" => skip_gt = true, + other => panic!("unknown arg: {other}"), + } + } + + let meta = fs::read_to_string(format!("{dir}/meta.json")).expect("meta.json"); + let (nb, nq, dim) = ( + meta_get(&meta, "n_base"), + meta_get(&meta, "n_query"), + meta_get(&meta, "dim"), + ); + println!("dataset: n={nb} queries={nq} dim={dim} (dir {dir})"); + + let base = load_f32(&format!("{dir}/base.f32"), nb, dim); + let queries = load_f32(&format!("{dir}/queries.f32"), nq, dim); + + // Ground truth: exact f32 brute force (same code shape as `real`), which + // doubles as the f32 scan-cost reference at this N. + let gt: Option>> = if skip_gt { + println!("ground truth: skipped (--skip-gt); recall columns omitted"); + None + } else { + let t0 = Instant::now(); + let gt: Vec> = queries + .iter() + .map(|q| { + let mut s: Vec<(f32, usize)> = base + .iter() + .enumerate() + .map(|(i, v)| (cosine(q, v), i)) + .collect(); + s.sort_by(|a, b| b.0.total_cmp(&a.0)); + s.truncate(10); + s.into_iter().map(|(_, i)| i).collect() + }) + .collect(); + let e = t0.elapsed(); + println!( + "ground truth: exact f32 cosine brute force {:.2?} total ({:.2} ms/q, single-thread f32 scan reference)", + e, + e.as_secs_f64() * 1e3 / nq as f64 + ); + Some(gt) + }; + let gts = gt.as_deref(); + + let mut rows: Vec = Vec::new(); + + // -- plain 5-bit (default) and working_dim-256, through mmap views ------ + for (bits, wd, name, note) in [ + (5u8, dim, "server_5bit_full", "default width"), + (5, 256, "server_5bit_wd256", "Matryoshka working_dim"), + (6, dim, "server_6bit_full", "recall lever"), + ] { + let (idx, build) = build_index(dim, wd, bits, &base); + let (path, file_bytes) = write_index(&dir, name, &idx); + drop(idx); + let (ms_q, r1, r10) = bench_view(&path, &queries, gts); + rows.push(Row { + mode: if wd == dim && bits == 5 { + "plain 5-bit view (default)".to_string() + } else if bits == 6 { + "plain 6-bit view".to_string() + } else { + format!("plain 5-bit view, wd={wd}") + }, + build: Some(build), + file_bytes, + ms_q, + recall1: r1, + recall10: r10, + note, + }); + } + + // -- plain 4-bit + residual (recall lever), through an mmap view -------- + { + let t0 = Instant::now(); + let mut idx = VecqIndex::with_residual(dim, 42); + for v in &base { + idx.add(v); + } + let build = t0.elapsed(); + let (path, file_bytes) = write_index(&dir, "server_resid_full", &idx); + drop(idx); + let (ms_q, r1, r10) = bench_view(&path, &queries, gts); + rows.push(Row { + mode: "plain 4-bit + residual view".to_string(), + build: Some(build), + file_bytes, + ms_q, + recall1: r1, + recall10: r10, + note: "recall lever", + }); + } + + // -- plain 4-bit view + cascade (#22) on the same persisted artifact ---- + // Methodology rule (see scale_probe): in-memory scoring (f32 scales) + // differs from the persisted f16 artifact at ~5e-4, which reorders + // tie-heavy top-10 lists. All rows are therefore measured against the + // SAME reloaded-from-file state so the columns are comparable. Cascade + // signatures are derived in memory from the reloaded codes (enable_cascade + // on a freshly loaded index) — same discipline as a serving process that + // mmaps the file, then enables the cascade. + let (idx4, build4) = build_index(dim, dim, 4, &base); + let (path4, file_bytes4) = write_index(&dir, "server_4bit_full", &idx4); + drop(idx4); + let (ms_q, r1, r10) = bench_view(&path4, &queries, gts); + rows.push(Row { + mode: "plain 4-bit view".to_string(), + build: Some(build4), + file_bytes: file_bytes4, + ms_q, + recall1: r1, + recall10: r10, + note: "cascade prefilter base", + }); + + // Reload the persisted (f16) artifact, then enable the cascade on it. + let raw4 = fs::read(&path4).expect("read 4-bit file"); + let mut idx4r = VecqIndex::from_bytes(&raw4).expect("parse 4-bit file"); + idx4r.enable_cascade(); + for &r in &[50usize, 100, 200, 400] { + let mut hits1 = 0usize; + let mut hits10 = 0usize; + let t = Instant::now(); + for (qi, q) in queries.iter().enumerate() { + let res = idx4r.search_cascade(q, 10, r); + if let Some(gt) = gts { + if res[0].0 == gt[qi][0] { + hits1 += 1; + } + for t in >[qi] { + if res.iter().any(|(i, _)| i == t) { + hits10 += 1; + } + } + } + } + let elapsed = t.elapsed(); + rows.push(Row { + mode: format!("cascade 4-bit r={r}"), + build: None, + file_bytes: file_bytes4, + ms_q: elapsed.as_secs_f64() * 1e3 / nq as f64, + recall1: gts.map(|_| hits1 as f32 / nq as f32), + recall10: gts.map(|_| hits10 as f32 / (nq * 10) as f32), + note: "signatures in RAM, not in file", + }); + } + + // -- table --------------------------------------------------------------- + println!( + "\n{:<28} {:>10} {:>10} {:>8} {:>9} {:>7} {:>8} note", + "mode", "build", "file MB", "B/vec", "ms/q", "r@1", "r@10" + ); + for row in &rows { + let build = row + .build + .map(|b| format!("{:.2?}", b)) + .unwrap_or_else(|| "—".into()); + let r1 = row + .recall1 + .map(|v| format!("{v:.3}")) + .unwrap_or_else(|| "—".into()); + let r10 = row + .recall10 + .map(|v| format!("{v:.3}")) + .unwrap_or_else(|| "—".into()); + println!( + "{:<28} {:>10} {:>10.1} {:>8} {:>9.2} {:>7} {:>8} {}", + row.mode, + build, + row.file_bytes as f64 / 1e6, + row.file_bytes / nb, + row.ms_q, + r1, + r10, + row.note + ); + } + println!( + "\nsingle-threaded, aarch64 release; recall deterministic, timings host-specific.\nExtends the edge profile (n=2000, docs/BENCHMARK.md) to server scale for #52." + ); +} diff --git a/crates/vecq-core/Cargo.toml b/crates/vecq-core/Cargo.toml index f77c05c..ffebad8 100644 --- a/crates/vecq-core/Cargo.toml +++ b/crates/vecq-core/Cargo.toml @@ -1,10 +1,13 @@ [package] name = "vecq-core" -description = "Training-free RHDH + Lloyd-Max 4-bit vector quantization with asymmetric scoring" +description = "Training-free vector quantization (4/5/6-bit) and search — the SQLite profile for edge vector storage" version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +homepage.workspace = true +keywords.workspace = true +categories.workspace = true authors.workspace = true readme = "../../README.md" exclude = ["benches"] diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index e250554..d74d6b3 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -4,7 +4,9 @@ Spike results, measured on aarch64 (Oracle ARM host), single-threaded, release p ## Setup -- Dataset: 2,000 base vectors + 100 queries, dim 768 +- Dataset: 2,000 base vectors + 100 queries, dim 768 (a second, server-scale + profile with 100,000 base vectors + 200 queries is documented in + [Server scale](#server-scale-issue-52) below) - Embeddings: real **EmbeddingGemma 300M** (Q4 ONNX) over a synthetic corpus of 18 topics × 10 modifiers (structured paragraphs, memory-note style) - Ground truth: exact f32 cosine brute-force @@ -41,6 +43,120 @@ All modes on the same dataset, aarch64 release, post wide-kernel: Split-layout codes (separate nibble/high-bit streams) project only ~2.2–2.7 ms/q — tracked with the full analysis in issue #40. +## Server scale (issue #52) + +Same pipeline as the edge profile, scaled to 100,000 base vectors + 200 +queries (dim 768), measured single-threaded on aarch64 (Oracle ARM host, +4 cores), release profile: + +| mode | build | file | B/vec | ms/q | recall@1 | recall@10 | +|---|---|---|---|---|---|---| +| f32 brute force (GT reference) | — | 307.2 MB | 3,072 | 59.8 | 1.000 (ref) | 1.000 (ref) | +| plain 5-bit view (default) | 3.8 s | 64.2 MB | 642 | 167.3 | 0.350 | 0.850 | +| plain 5-bit view, wd=256 | 1.0 s | 16.2 MB | 162 | 41.5 | 0.400 | 0.772 | +| plain 6-bit view | 4.3 s | 77.0 MB | 770 | 170.6 | 0.350 | 0.856 | +| plain 4-bit + residual view | 6.3 s | 102.8 MB | 1,028 | 98.8 | 0.320 | 0.836 | +| plain 4-bit view | 3.5 s | 51.4 MB | 514 | 48.0 | 0.355 | 0.818 | +| cascade 4-bit r=50 | — | 51.4 MB | 514 | 74.5 | 0.375 | 0.817 | +| cascade 4-bit r=100 | — | 51.4 MB | 514 | 75.2 | 0.380 | 0.817 | +| cascade 4-bit r=200 | — | 51.4 MB | 514 | 72.4 | 0.345 | 0.818 | +| cascade 4-bit r=400 | — | 51.4 MB | 514 | 76.3 | 0.345 | 0.818 | + +Reproduce: + +```sh +python3 scripts/gen_dataset.py --n-base 100000 --n-query 200 --out /tmp/vecq-bench-100k +cargo run --release -p vecq-bench --bin server_scale +``` + +Methodology (honest labeling): + +- The dataset is seeded and reproducible; the corpus builder is byte-identical + to the edge profile's, so the server base set is a strict superset of the + edge one. The published run embedded the corpus on x86 (Modal CPU workers, + onnxruntime 1.25.0) using the same model files and the same generator code — + embeddings from the two runtimes agree at cosine ≥ 0.9996 on a 64-vector + probe — while every vecq-side number (quantization, search, ground truth) + is measured on the aarch64 host against exactly this persisted dataset. +- All rows are scored against the **persisted f16 artifact** (mmap'd view or + file-reloaded index), not in-memory f32-scale state: f16 scales perturb + scores by ≤ ~5e-4, which reorders tie-heavy top-10 lists, so mixing the two + representations makes columns incomparable (`scale_probe` harness documents + the delta). Cascade signatures are derived in memory from the reloaded + codes, as a serving process would. +- Recall values are deterministic for a given dataset; timings are + host-specific. The edge-profile recall tables are unaffected. + +Findings: + +- **Storage and build scale linearly and hold**: 4.78x compression at 100K + (64.2 MB vs 307.2 MB), 4.3 s single-threaded build. The compression story + does not degrade with N — a 100K × 768 index fits in ~64 MB of RAM or page + cache. +- **Recall is N-dependent**: with 50x more data the true top-10 neighbors sit + much closer together, and ~1e-3 quantization noise flips rankings — 5-bit + recall@10 falls 0.974 → 0.850 and recall@1 0.940 → 0.350 (4-bit: 0.957 → + 0.818). Where recall at server N matters, the available levers are 6-bit / + residual (edge-profile recall advantage carries structurally, at their + storage cost) — measure on your corpus before committing. +- **Single-thread scan at server N**: 5/6-bit scoring is extraction-bound and + at 100K loses to exact f32 brute force (167/171 vs 60 ms/q; the residual + mode at 99 ms/q also trails), while only 4-bit (48 ms/q) and wd=256 + (42 ms/q) stay ahead of the f32 scan. vecq's server pitch at this N is the + footprint, not raw single-thread latency. +- **Cascade (#22) is not the server-scale throughput lever**: prefilter + r + rescore costs as much as the whole plain 4-bit scan (72–76 vs 48 ms/q) at + equal recall (r ≥ 200 saturates to plain). The remaining lever is a + parallel scan (#51): same kernels over chunks + fixed-order merge. +- **wd=256 (Matryoshka truncation)**: 19x compression vs f32 (162 B/vec) and + the fastest scan, but recall on this 18-topic corpus is materially lower + (r@10 0.772) — truncation quality is corpus-dependent, evaluate per + workload. + +### Scale curve and positioning + +Same dataset, identical queries, ground truth recomputed per prefix. The +10K point runs the same `server_scale` harness over the first 10,000 base +vectors; recall values are deterministic (identical across runs), timings +are from a quiet host: + +**10,000 vectors (local/on-device scale):** + +| mode | build | B/vec | ms/q | recall@1 | recall@10 | +|---|---|---|---|---|---| +| plain 5-bit view (default) | 0.38 s | 642 | 16.5 | 0.805 | 0.932 | +| plain 5-bit view, wd=256 | 0.10 s | 162 | 4.1 | 0.735 | 0.831 | +| plain 6-bit view | 0.42 s | 770 | 16.7 | 0.795 | 0.949 | +| plain 4-bit + residual view | 0.59 s | 1,028 | 9.0 | 0.810 | 0.951 | +| plain 4-bit view | 0.34 s | 514 | 4.5 | 0.790 | 0.914 | +| f32 brute force (GT reference) | — | 3,072 | 5.9 | 1.000 (ref) | 1.000 (ref) | + +The edge-profile mode hierarchy carries over intact: 6-bit and residual +remain the recall levers (r@10 0.95 at 10K), residual stays the fastest +high-recall path, and the compression story holds at every width. + +**Scale curve, 5-bit default:** + +| N | recall@1 | recall@10 | ms/q | file | +|---|---|---|---|---| +| 2,000 | 0.940 | 0.977 | 3.2 | 1.3 MB | +| 10,000 | 0.805 | 0.932 | 16.5 | 6.4 MB | +| 100,000 | 0.350 | 0.850 | 167.3 | 64.2 MB | + +Positioning (deliberate, measured): vecq's sweet spot is the **local / +on-device profile — up to roughly 10K vectors** — where it keeps recall@10 +≥ 0.93 at 4.78x compression with interactive single-thread latency (and +≥ 0.95 with the residual recall lever). Beyond +that, the constraint is not scan speed but neighbor geometry: as N grows the +true top-10 margins collapse (avg 10th-vs-11th margin 2.0e-3 at 2K vs +3.6e-4 at 100K on this corpus), so ~1e-3 quantization noise reorders +identity-level rankings. Notably, when vecq's top-1 differs from the exact +top-1 at 100K, the f32 quality gap is tiny (avg 2.1e-5) — the returned +neighbor is nearly as good, just not the same vector. Workloads that need +strict identity recall at 10^5+ vectors should use f32 or a trained +higher-precision index; a parallel scan (#51) would fix latency at server N +but not this recall ceiling. + ## Changelog vs first spike measurement - **Search 1.75x faster** (3.32 → 0.89 ms/q after NEON + batching): the scoring loop now uses a diff --git a/scripts/gen_dataset.py b/scripts/gen_dataset.py new file mode 100644 index 0000000..5ecdb17 --- /dev/null +++ b/scripts/gen_dataset.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Generate the real EmbeddingGemma dataset used by the vecq benchmarks. + +Two documented profiles (see docs/BENCHMARK.md): + + edge (defaults) --n-base 2000 --n-query 100 --out /tmp/vecq-bench + server --n-base 100000 --n-query 200 --out /tmp/vecq-bench-100k + +The corpus builder is byte-identical to the original spike generator, so the +server-scale base vectors are a strict superset of the edge ones (same seed 1 +draw sequence) and both profiles share one methodology: real EmbeddingGemma +300M (Q4 ONNX) embeddings over a synthetic corpus of 18 topics x 10 modifiers, +mean-pooled and L2-normalized. + +Requires: onnxruntime, tokenizers, numpy (bench-only; vecq-core stays zero-dep). +""" +import argparse +import json +import os + +import numpy as np +import onnxruntime as ort +from tokenizers import Tokenizer + +TOPICS = [ + "rust memory safety", "flutter state management", "vector databases", + "mobile battery optimization", "sqlite on android", "machine learning inference", + "indonesian street food", "coffee brewing methods", "mountain hiking gear", + "startup funding basics", "customer retention tactics", "email deliverability", + "quantum computing basics", "solar panel efficiency", "electric vehicle charging", + "jazz music theory", "film editing techniques", "urban gardening", +] +MODIFIERS = [ + "for beginners", "in production", "common pitfalls", "advanced guide", + "case study", "best practices", "2026 update", "lessons learned", + "performance notes", "checklist", +] + + +def sample_texts(n, seed): + rng = np.random.default_rng(seed) + texts = [] + for i in range(n): + t = TOPICS[rng.integers(len(TOPICS))] + m = MODIFIERS[rng.integers(len(MODIFIERS))] + body = ( + f"Notes on {t} {m}. " + f"The key insight about {t} involves careful attention to detail and repeated practice. " + f"When working with {t}, engineers often observe measurable improvements after iteration. " + f"Chapter {i % 97}: applications of {t} {m} in real projects show consistent results " + f"across datasets and environments, with variance depending on configuration choices." + ) + texts.append(body) + return texts + + +def embed(texts, sess, tok, batch=64): + embs = [] + n_batches = (len(texts) + batch - 1) // batch + for s in range(0, len(texts), batch): + chunk = texts[s:s + batch] + enc = tok.encode_batch([f"title: none\ntext: {t}" for t in chunk]) + # dynamic max len for this batch + maxlen = min(2048, max(len(e.ids) for e in enc)) + ids = np.zeros((len(chunk), maxlen), dtype=np.int64) + mask = np.zeros((len(chunk), maxlen), dtype=np.int64) + for i, e in enumerate(enc): + ids[i, :len(e.ids)] = e.ids[:maxlen] + mask[i, :len(e.ids)] = e.attention_mask[:maxlen] + out = sess.run(None, {"input_ids": ids, "attention_mask": mask})[0] + m = mask[:, :, None].astype(np.float32) + emb = (out * m).sum(1) / np.clip(m.sum(1), 1e-9, None) + emb = emb / np.linalg.norm(emb, axis=1, keepdims=True) + embs.append(emb.astype(np.float32)) + done = min(s + batch, len(texts)) + if (s // batch) % 16 == 0 or done == len(texts): + print(f" embedded {done}/{len(texts)} (batch {s // batch + 1}/{n_batches})", + flush=True) + return np.concatenate(embs) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--n-base", type=int, default=2000) + ap.add_argument("--n-query", type=int, default=100) + ap.add_argument("--base-seed", type=int, default=1) + ap.add_argument("--query-seed", type=int, default=2) + ap.add_argument("--model", default="/opt/data/models/embeddinggemma-q4/onnx/model_q4.onnx") + ap.add_argument("--tokenizer", default="/opt/data/models/embeddinggemma-q4/tokenizer.json") + ap.add_argument("--out", default="/tmp/vecq-bench") + args = ap.parse_args() + + os.makedirs(args.out, exist_ok=True) + tok = Tokenizer.from_file(args.tokenizer) + sess = ort.InferenceSession(args.model, providers=["CPUExecutionProvider"]) + print("input:", [i.name for i in sess.get_inputs()], flush=True) + base = embed(sample_texts(args.n_base, args.base_seed), sess, tok) + queries = embed(sample_texts(args.n_query, args.query_seed), sess, tok) + print("base:", base.shape, "queries:", queries.shape) + base.tofile(f"{args.out}/base.f32") + queries.tofile(f"{args.out}/queries.f32") + with open(f"{args.out}/meta.json", "w") as f: + json.dump({ + "n_base": args.n_base, + "n_query": args.n_query, + "dim": int(base.shape[1]), + "base_seed": args.base_seed, + "query_seed": args.query_seed, + }, f) + print("written to", args.out) + + +if __name__ == "__main__": + main()