diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d6b64d..365b7e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,48 @@ +## [0.9.2] - 2026-08-25 + +### 🎯 Release Overview +Bug-fix release. Two search failures that hit ordinary queries, and a +concurrency crash between the daemon and foreground commands. No new features, +no CLI changes, no migration required. + +### 🐛 Fixes + +#### Search no longer errors on punctuated queries +- `smriti search node-llama-cpp` failed with `no such column: llama`; + `smriti search "qmd 2.8.3"` failed with `fts5: syntax error near "."`. + `searchFiltered` interpolated the raw query into the FTS5 MATCH expression, + so punctuation was parsed as FTS5 grammar rather than as text. Hyphenated + names are common in these transcripts, so this hit everyday searches. +- The recall path had the matching defect more quietly: `buildMemoryFTS5Query` + stripped punctuation instead of splitting on it, collapsing `2026.4.10` to + `2026410` while the index held `2026`, `4`, `10`. Silent zero results. +- Both paths now split on the boundaries the `porter unicode61` tokenizer + uses and AND the parts, so a query tokenizes the way the indexed text did. + +#### Daemon no longer races foreground commands into "database is locked" +- `createStore()` runs the WAL migration and FTS trigger DDL before returning, + so the `busy_timeout` set on the returned handle arrived too late and that + DDL raced at bun:sqlite's default of 0. The daemon re-runs `createStore()` + per flush, so any overlap with a foreground `recall` / `embed` / `ingest` + failed on contact. +- QMD's `openDatabase()` now owns both `busy_timeout` (120s default, + `QMD_SQLITE_BUSY_TIMEOUT` to override) and a retrying WAL migration. +- Measured: six concurrent cold opens on one fresh DB went from **1/6** + succeeding to **6/6**. + +### 🔧 Dependencies +- QMD submodule synced to upstream `v2.8.3` (155 commits, from `v2.1.0`). + Zero breaking changes to the APIs Smriti consumes. `content_vectors` gains + an `embed_fingerprint` column, populated automatically for new vectors; + existing embeddings are unaffected and no re-embed is needed. +- QMD's embedding runtime moves to node-llama-cpp 3.20.0 (llama.cpp b10361). + +### 📝 Notes +- `CHANGELOG.md` has no entries for 0.7.0 through 0.9.1; those releases were + tagged by the auto-release job but never written up here. `package.json` also + drifted — it still read `0.9.0` at the `v0.9.1` tag, because auto-release + tags without bumping the manifest. This release realigns them. + ## [0.6.0] - 2026-03-14 ### 🎯 Release Overview diff --git a/package.json b/package.json index ef2b85c..8593d08 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "smriti", - "version": "0.9.0", + "version": "0.9.2", "description": "Smriti - Unified memory layer across all AI agents", "type": "module", "bin": { diff --git a/qmd b/qmd index da67604..c1e8f6f 160000 --- a/qmd +++ b/qmd @@ -1 +1 @@ -Subproject commit da67604ac32f48d58177311db4f92e062d883af1 +Subproject commit c1e8f6f60d8a33cf4c74720859a6116aa1845820 diff --git a/src/db.ts b/src/db.ts index b9610a4..ae5d18a 100644 --- a/src/db.ts +++ b/src/db.ts @@ -730,11 +730,12 @@ export async function initSmriti(dbPath?: string): Promise { setQmdStore(store); const db = store.internal.db as unknown as Database; _db = db; - // busy_timeout is per-connection, not persisted in the DB file — set it on - // every open. Without it, two processes opening the same SQLite file at - // once (e.g. the daemon's flush and a manual `smriti ingest --force`) fail - // immediately with "database is locked" instead of retrying briefly. - db.exec("PRAGMA busy_timeout = 5000"); + // busy_timeout and WAL are set by QMD's openDatabase() inside createStore(), + // which is the only place that runs *before* the cold-open schema work + // (WAL migration, FTS trigger DDL). Setting them here instead was too late: + // that DDL raced with busy_timeout still at bun:sqlite's default of 0, so a + // daemon flush overlapping a foreground command failed on contact rather + // than queueing. Override with QMD_SQLITE_BUSY_TIMEOUT (ms) if needed. initializeMemoryTables(db as any); initializeSmritiTables(db); seedDefaults(db); diff --git a/src/memory.ts b/src/memory.ts index f5045c1..6f6a4e3 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -147,17 +147,23 @@ export function initializeMemoryTables(db: Database): void { // FTS5 Query Building (same pattern as store.ts buildFTS5Query) // ============================================================================= -function sanitizeMemoryFTSTerm(term: string): string { - return term.replace(/[^\p{L}\p{N}']/gu, "").toLowerCase(); -} - -function buildMemoryFTS5Query(query: string): string | null { +/** + * Build an FTS5 MATCH expression from a user query. + * + * Splits on everything the `porter unicode61` tokenizer treats as a boundary, + * so a query tokenizes the same way the indexed text did. Without this, + * punctuation either matched nothing ("2026.4.10" collapsed to "2026410", but + * the index holds "2026"/"4"/"10") or leaked into FTS5's own grammar + * ("node-llama-cpp" parsed "llama" as a column name). + * + * Exported so `searchFiltered` builds its MATCH the same way. + */ +export function buildMemoryFTS5Query(query: string): string | null { const terms = query - .split(/\s+/) - .map((t) => sanitizeMemoryFTSTerm(t)) + .split(/[^\p{L}\p{N}']+/u) + .map((t) => t.toLowerCase()) .filter((t) => t.length > 0); if (terms.length === 0) return null; - if (terms.length === 1) return `"${terms[0]}"*`; return terms.map((t) => `"${t}"*`).join(" AND "); } diff --git a/src/search/index.ts b/src/search/index.ts index 2021fa3..3b5eeaf 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -8,6 +8,7 @@ import type { Database } from "bun:sqlite"; import { DEFAULT_SEARCH_LIMIT } from "../config"; import { searchMemoryFTS, searchMemoryVec } from "../qmd"; +import { buildMemoryFTS5Query } from "../memory"; // ============================================================================= // Types @@ -74,9 +75,15 @@ export function searchFiltered( const conditions: string[] = []; const params: any[] = []; - // FTS match condition with column filter + // FTS match condition with column filter. The query is tokenized the same + // way the index was (see buildMemoryFTS5Query) rather than interpolated raw + // — raw user input was parsed as FTS5 grammar, so "node-llama-cpp" failed + // with `no such column: llama` and "2.8.3" with `syntax error near "."`. + // Parenthesised so the column filter binds to the whole AND group. + const ftsQuery = buildMemoryFTS5Query(query); + if (!ftsQuery) return []; conditions.push(`memory_fts MATCH ?`); - params.push(`{${columns.join(" ")}} : ${query}`); + params.push(`{${columns.join(" ")}} : (${ftsQuery})`); // Category filter if (filters.category) { diff --git a/test/search.test.ts b/test/search.test.ts index 18d2334..7768e3e 100644 --- a/test/search.test.ts +++ b/test/search.test.ts @@ -13,6 +13,7 @@ import { insertVoiceNote, } from "../src/db"; import { searchFiltered, listSessions } from "../src/search/index"; +import { buildMemoryFTS5Query } from "../src/memory"; let db: Database; @@ -77,7 +78,8 @@ beforeAll(() => { ('s3', 'user', 'The login page has an error when submitting', 'h5', '${now}'), ('s3', 'assistant', 'Fixed the login bug by validating input', 'h6', '${now}'), ('s4', 'user', 'Help me build a tax calculator', 'h7', '${now}'), - ('s4', 'assistant', 'Here is a tax calculator implementation', 'h8', '${now}'); + ('s4', 'assistant', 'Here is a tax calculator implementation', 'h8', '${now}'), + ('s1', 'user', 'Bumped qmd to 2026.4.10 using node-llama-cpp and sqlite-vec', 'h9', '${now}'); `); // Insert sidecar content for s4 (claude-web session) @@ -245,3 +247,33 @@ test("migrateFTSToV2 is idempotent", () => { const results = searchFiltered(db, "calculateTax"); expect(results.length).toBeGreaterThan(0); }); + +// Regression: user queries were interpolated into the FTS5 MATCH expression +// verbatim, so punctuation was parsed as FTS5 grammar rather than tokenized. +// "node-llama-cpp" raised `no such column: llama` and "2026.4.10" raised +// `fts5: syntax error near "."`. Mirrors QMD upstream #563. + +test("searchFiltered matches hyphenated terms instead of erroring", () => { + const results = searchFiltered(db, "node-llama-cpp", { limit: 10 }); + expect(results.length).toBeGreaterThan(0); + expect(results.some((r) => r.session_id === "s1")).toBe(true); +}); + +test("searchFiltered matches dotted version strings", () => { + const results = searchFiltered(db, "2026.4.10", { limit: 10 }); + expect(results.length).toBeGreaterThan(0); + expect(results.some((r) => r.session_id === "s1")).toBe(true); +}); + +test("searchFiltered still narrows — punctuated queries are ANDed, not dropped", () => { + expect(searchFiltered(db, "sqlite-vec", { limit: 10 }).length).toBeGreaterThan(0); + expect(searchFiltered(db, "nothing-here-xyz", { limit: 10 }).length).toBe(0); +}); + +test("buildMemoryFTS5Query tokenizes on the same boundaries as the index", () => { + expect(buildMemoryFTS5Query("node-llama-cpp")).toBe('"node"* AND "llama"* AND "cpp"*'); + expect(buildMemoryFTS5Query("2026.4.10")).toBe('"2026"* AND "4"* AND "10"*'); + expect(buildMemoryFTS5Query("plain")).toBe('"plain"*'); + expect(buildMemoryFTS5Query(" ")).toBe(null); + expect(buildMemoryFTS5Query("!!!")).toBe(null); +});