Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 77 additions & 89 deletions Cargo.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ serde-wasm-bindgen = { version = "0.6", optional = true }
js-sys = { version = "0.3", optional = true }
web-sys = { version = "0.3", optional = true, features = ["console", "Window", "Storage", "IdbFactory", "IdbDatabase", "IdbObjectStore", "IdbTransaction"] }
console_error_panic_hook = { version = "0.1", optional = true }
parquet = { version = "57", optional = true }
arrow = { version = "57", optional = true }
parquet = { version = "59", optional = true }
arrow = { version = "59", optional = true }
candle-core = { version = "0.9", optional = true }
candle-nn = { version = "0.9", optional = true }
candle-transformers = { version = "0.9", optional = true }
Expand Down
2 changes: 1 addition & 1 deletion benches/openai_embeddings_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
//
// Run with: cargo bench --features "embeddings,openai-embeddings" --bench openai_embeddings_bench

use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
use vecstore::embeddings::openai_backend::{OpenAIEmbedding, OpenAIModel};

// Helper to create embedder
Expand Down
4 changes: 2 additions & 2 deletions benches/rag_benchmarks.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
use std::collections::HashMap;
use vecstore::{
Metadata, Query, VecStore,
rag_utils::MultiQueryRetrieval,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Query, VecStore,
};

// Mock embedding function for benchmarks
Expand Down
2 changes: 1 addition & 1 deletion benches/simd_bench.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
use vecstore::simd::*;

fn generate_random_vector(dim: usize) -> Vec<f32> {
Expand Down
4 changes: 2 additions & 2 deletions benches/vecstore_bench.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
use std::collections::HashMap;
use vecstore::{make_record, FilterExpr, FilterOp, Metadata, Query, VecStore};
use vecstore::{FilterExpr, FilterOp, Metadata, Query, VecStore, make_record};

fn setup_store_with_data(n: usize, dim: usize) -> (tempfile::TempDir, VecStore) {
let temp_dir = tempfile::tempdir().unwrap();
Expand Down
6 changes: 3 additions & 3 deletions benches/wasm_hnsw_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
//
// Run with: cargo bench --bench wasm_hnsw_bench

use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
use rand::Rng;
use vecstore::store::wasm_hnsw::WasmHnsw;
use vecstore::Distance;
use vecstore::store::wasm_hnsw::WasmHnsw;

fn generate_random_vectors(n: usize, dim: usize) -> Vec<Vec<f32>> {
let mut rng = rand::thread_rng();
(0..n)
.map(|_| (0..dim).map(|_| rng.gen::<f32>()).collect())
.map(|_| (0..dim).map(|_| rng.r#gen::<f32>()).collect())
.collect()
}

Expand Down
2 changes: 1 addition & 1 deletion examples/01_basic_rag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
use anyhow::Result;
use std::collections::HashMap;
use vecstore::{
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Query, VecStore,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
};

fn main() -> Result<()> {
Expand Down
2 changes: 1 addition & 1 deletion examples/03_web_scraping_rag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
use anyhow::Result;
use std::collections::HashMap;
use vecstore::{
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Query, VecStore,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
};

fn main() -> Result<()> {
Expand Down
2 changes: 1 addition & 1 deletion examples/04_code_search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
use anyhow::Result;
use std::collections::HashMap;
use vecstore::{
text_splitter::{CodeTextSplitter, TextSplitter},
Metadata, Query, VecStore,
text_splitter::{CodeTextSplitter, TextSplitter},
};

fn main() -> Result<()> {
Expand Down
2 changes: 1 addition & 1 deletion examples/06_reranking_pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
use anyhow::Result;
use std::collections::HashMap;
use vecstore::{
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Neighbor, Query, VecStore,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
};

fn main() -> Result<()> {
Expand Down
2 changes: 1 addition & 1 deletion examples/07_multi_query_rag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
use anyhow::Result;
use std::collections::HashMap;
use vecstore::{
Metadata, Query, VecStore,
rag_utils::MultiQueryRetrieval,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Query, VecStore,
};

fn main() -> Result<()> {
Expand Down
2 changes: 1 addition & 1 deletion examples/08_conversation_rag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
use anyhow::Result;
use std::collections::HashMap;
use vecstore::{
Metadata, Query, VecStore,
rag_utils::ConversationMemory,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Query, VecStore,
};

fn main() -> Result<()> {
Expand Down
7 changes: 5 additions & 2 deletions examples/09_evaluation_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
use anyhow::Result;
use std::collections::HashMap;
use vecstore::{
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Query, VecStore,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
};

fn main() -> Result<()> {
Expand Down Expand Up @@ -50,7 +50,10 @@ fn main() -> Result<()> {
"What is VecStore?",
"VecStore is a high-performance vector database built in Rust.",
),
("How fast is VecStore?", "Rust avoids the Python interpreter in tight loops."),
(
"How fast is VecStore?",
"Rust avoids the Python interpreter in tight loops.",
),
];

let mut total_score = 0.0;
Expand Down
2 changes: 1 addition & 1 deletion examples/10_production_rag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ use anyhow::{Context, Result};
use std::collections::HashMap;
use std::time::Instant;
use vecstore::{
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
Metadata, Query, VecDatabase,
text_splitter::{RecursiveCharacterTextSplitter, TextSplitter},
};

fn main() -> Result<()> {
Expand Down
2 changes: 1 addition & 1 deletion examples/bulk_migration_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
use anyhow::Result;
use std::fs;
use tempfile::TempDir;
use vecstore::bulk_migration::{ChromaDBMigration, MigrationConfig, PineconeMigration};
use vecstore::VecStore;
use vecstore::bulk_migration::{ChromaDBMigration, MigrationConfig, PineconeMigration};

fn main() -> Result<()> {
println!("🔄 VecStore Bulk Migration Demo\n");
Expand Down
4 changes: 2 additions & 2 deletions examples/compression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
//! ```

use vecstore::compression::{
decode_rle, encode_rle, encode_varint, CompressedNeighborList,
CompressionConfig, CompressionLevel, CompressionStats,
CompressedNeighborList, CompressionConfig, CompressionLevel, CompressionStats, decode_rle,
encode_rle, encode_varint,
};

fn main() {
Expand Down
2 changes: 1 addition & 1 deletion examples/dim_reduction_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
//! - Reconstruct original vectors

use anyhow::Result;
use vecstore::dim_reduction::{ReductionStats, PCA};
use vecstore::dim_reduction::{PCA, ReductionStats};

fn generate_high_dim_data() -> Vec<Vec<f32>> {
let mut vectors = Vec::new();
Expand Down
4 changes: 2 additions & 2 deletions examples/distance_metrics_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@

use std::collections::HashMap;
use vecstore::{
hamming_distance_simd, jaccard_distance_simd, manhattan_distance_simd, Distance, Metadata,
VecStore,
Distance, Metadata, VecStore, hamming_distance_simd, jaccard_distance_simd,
manhattan_distance_simd,
};

fn main() -> anyhow::Result<()> {
Expand Down
4 changes: 2 additions & 2 deletions examples/distributed_indexing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
//! ```

use vecstore::distributed::{
ConsistencyLevel, ConsistentHashRing, DistributedConfig, DistributedStore,
ReplicationStrategy, ShardingStrategy,
ConsistencyLevel, ConsistentHashRing, DistributedConfig, DistributedStore, ReplicationStrategy,
ShardingStrategy,
};

fn main() {
Expand Down
2 changes: 1 addition & 1 deletion examples/embedding_integration_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

#[cfg(feature = "embeddings")]
use vecstore::{
embeddings::{EmbeddingCollection, SimpleEmbedder, TextEmbedder},
Metadata, VecDatabase,
embeddings::{EmbeddingCollection, SimpleEmbedder, TextEmbedder},
};

#[cfg(feature = "embeddings")]
Expand Down
4 changes: 2 additions & 2 deletions examples/fuzzy_search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
//! ```

use vecstore::fuzzy::{
damerau_levenshtein_distance, levenshtein_distance, similarity_score, suggest_corrections,
BKTree, FuzzyMatcher,
BKTree, FuzzyMatcher, damerau_levenshtein_distance, levenshtein_distance, similarity_score,
suggest_corrections,
};

fn main() {
Expand Down
4 changes: 2 additions & 2 deletions examples/gpu_acceleration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -294,10 +294,10 @@ fn main() {
};

println!("{:<20} {:>12.2} {:>14.2}x", name, duration_ms, speedup);
}
},
Err(_) => {
println!("{:<20} {:>12} {:>15}", name, "N/A", "N/A");
}
},
}
}

Expand Down
4 changes: 2 additions & 2 deletions examples/graph_visualization.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,14 @@ fn main() -> Result<()> {
println!(" 2. Open graph.json in a D3.js visualization tool");
println!(" 3. Load graph_cytoscape.json in Cytoscape.js");
println!(" 4. Read graph_stats.txt for detailed metrics");
}
},
Err(e) => {
println!("⚠️ Graph visualization not available: {}", e);
println!("\nNote: Graph visualization is currently only supported for WASM builds.");
println!("To use visualization:");
println!(" 1. Build with: cargo build --target wasm32-unknown-unknown");
println!(" 2. Or use the WASM-based backend");
}
},
}

Ok(())
Expand Down
4 changes: 2 additions & 2 deletions examples/langchain_rag_pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@
use anyhow::Result;
use vecstore::error::VecStoreError;
use vecstore::{
print_health_report, Document, HealthChecker, LangChainVectorStore, RetrieverConfig, VecStore,
VectorStoreRetriever,
Document, HealthChecker, LangChainVectorStore, RetrieverConfig, VecStore, VectorStoreRetriever,
print_health_report,
};

fn main() -> Result<()> {
Expand Down
4 changes: 3 additions & 1 deletion examples/metadata_indexing_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,9 @@ fn main() -> Result<()> {
println!(" • Range queries: O(log N) instead of O(N)");
println!(" • Equality queries: O(1) instead of O(N)");
println!(" • Compound queries: Fast intersection of index results");
println!(" • Potential speedup: orders of magnitude for large datasets (measure on your workload)");
println!(
" • Potential speedup: orders of magnitude for large datasets (measure on your workload)"
);

println!("\n📝 Use Cases:");
println!(" • E-commerce: Filter by price, category, rating");
Expand Down
2 changes: 1 addition & 1 deletion examples/namespace_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ fn main() -> anyhow::Result<()> {
match result {
Err(e) if e.to_string().contains("not active") => {
println!(" ✅ Write rejected: {}", e);
}
},
_ => println!(" ⚠️ Expected error but got: {:?}", result),
}

Expand Down
2 changes: 1 addition & 1 deletion examples/openai_basic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ async fn main() -> Result<()> {
eprintln!("\nPlease set your OpenAI API key:");
eprintln!(" export OPENAI_API_KEY='your-api-key-here'\n");
std::process::exit(1);
}
},
};

println!("✓ API key found\n");
Expand Down
6 changes: 4 additions & 2 deletions examples/openai_cost_tracking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ async fn main() -> Result<()> {
eprintln!("\nPlease set your OpenAI API key:");
eprintln!(" export OPENAI_API_KEY='your-api-key-here'\n");
std::process::exit(1);
}
},
};

println!("✓ API key found\n");
Expand Down Expand Up @@ -378,6 +378,8 @@ fn format_number(n: usize) -> String {
#[cfg(not(all(feature = "embeddings", feature = "openai-embeddings")))]
fn main() {
eprintln!("This example requires the 'embeddings' and 'openai-embeddings' features.");
eprintln!("Run with: cargo run --example openai_cost_tracking --features \"embeddings,openai-embeddings\"");
eprintln!(
"Run with: cargo run --example openai_cost_tracking --features \"embeddings,openai-embeddings\""
);
std::process::exit(1);
}
37 changes: 29 additions & 8 deletions examples/openai_rag.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,21 +38,42 @@ async fn main() -> Result<()> {
eprintln!("\nPlease set your OpenAI API key:");
eprintln!(" export OPENAI_API_KEY='your-api-key-here'\n");
std::process::exit(1);
}
},
};

println!("✓ API key found\n");

// 2. Create sample documents to index
println!("Creating sample documents...");
let documents = vec![
("rust_intro", "Rust is a systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety. It accomplishes these goals without using a garbage collector."),
("rust_ownership", "Rust's ownership system is its most unique feature. The ownership rules ensure memory safety without needing a garbage collector. Each value has a variable that's its owner, and there can only be one owner at a time."),
("rust_cargo", "Cargo is Rust's build system and package manager. It handles building your code, downloading dependencies, and building those dependencies. Most Rustaceans use Cargo to manage their Rust projects."),
("python_intro", "Python is a high-level, interpreted programming language known for its simplicity and readability. It supports multiple programming paradigms and has a comprehensive standard library."),
("python_django", "Django is a high-level Python web framework that encourages rapid development and clean, pragmatic design. It follows the model-template-views architectural pattern and includes an ORM."),
("javascript_intro", "JavaScript is a versatile programming language primarily used for web development. It runs in browsers and enables interactive web pages. Node.js allows JavaScript to run on servers too."),
("vecstore_intro", "VecStore is a high-performance vector database written in Rust. It provides efficient similarity search using HNSW indexing, supports multiple embedding backends, and offers both Rust and Python APIs."),
(
"rust_intro",
"Rust is a systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety. It accomplishes these goals without using a garbage collector.",
),
(
"rust_ownership",
"Rust's ownership system is its most unique feature. The ownership rules ensure memory safety without needing a garbage collector. Each value has a variable that's its owner, and there can only be one owner at a time.",
),
(
"rust_cargo",
"Cargo is Rust's build system and package manager. It handles building your code, downloading dependencies, and building those dependencies. Most Rustaceans use Cargo to manage their Rust projects.",
),
(
"python_intro",
"Python is a high-level, interpreted programming language known for its simplicity and readability. It supports multiple programming paradigms and has a comprehensive standard library.",
),
(
"python_django",
"Django is a high-level Python web framework that encourages rapid development and clean, pragmatic design. It follows the model-template-views architectural pattern and includes an ORM.",
),
(
"javascript_intro",
"JavaScript is a versatile programming language primarily used for web development. It runs in browsers and enables interactive web pages. Node.js allows JavaScript to run on servers too.",
),
(
"vecstore_intro",
"VecStore is a high-performance vector database written in Rust. It provides efficient similarity search using HNSW indexing, supports multiple embedding backends, and offers both Rust and Python APIs.",
),
];

println!("✓ Created {} documents", documents.len());
Expand Down
2 changes: 1 addition & 1 deletion examples/performance_monitoring.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
//! - Resource utilization

use anyhow::Result;
use vecstore::{print_health_report, BenchmarkConfig, Benchmarker, HealthChecker, VecStore};
use vecstore::{BenchmarkConfig, Benchmarker, HealthChecker, VecStore, print_health_report};

fn main() -> Result<()> {
println!("📊 VecStore Performance Monitoring Example\n");
Expand Down
1 change: 0 additions & 1 deletion examples/phrase_matching_demo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
//!
//! Run with: cargo run --example phrase_matching_demo


// Mock TextIndex for demonstration (in practice, use VecStore's internal TextIndex)
fn main() {
println!("=== VecStore Phrase Matching Demo ===\n");
Expand Down
Loading
Loading