From 79f36cf22f6c250eaae9c86273da6e112181d3bc Mon Sep 17 00:00:00 2001 From: Ashutosh Tripathi Date: Tue, 25 Aug 2026 12:47:12 +0530 Subject: [PATCH 1/3] fix(db): let QMD own busy_timeout and WAL so cold-open DDL is protected createStore() runs the WAL migration and the FTS trigger DDL before it returns, so the busy_timeout Smriti set on the returned handle arrived after the only section that needed it. That DDL raced at bun:sqlite's default of 0. The daemon makes this reachable rather than theoretical: defaultFlushAgent opens the store, ingests and closes it per flush, so a watching daemon re-runs createStore() on every debounced change. flushChain serializes flushes only within the daemon process, and there is no cross-process lock, so any overlap with a foreground recall/embed/ingest failed on contact with 'database is locked'. Upstream moved busy_timeout and a retrying WAL migration into openDatabase(), the only place that runs before the DDL. Bumps the qmd submodule to that (155 commits, v2.1.0 -> v2.8.3). Six concurrent cold opens on one fresh DB: 1/6 succeeded before, 6/6 after. Claude-Session: https://claude.ai/code/session_01HWzCvpW1yLZD3SJTXEojvR --- qmd | 2 +- src/db.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/qmd b/qmd index da67604..c1e8f6f 160000 --- a/qmd +++ b/qmd @@ -1 +1 @@ -Subproject commit da67604ac32f48d58177311db4f92e062d883af1 +Subproject commit c1e8f6f60d8a33cf4c74720859a6116aa1845820 diff --git a/src/db.ts b/src/db.ts index b9610a4..ae5d18a 100644 --- a/src/db.ts +++ b/src/db.ts @@ -730,11 +730,12 @@ export async function initSmriti(dbPath?: string): Promise { setQmdStore(store); const db = store.internal.db as unknown as Database; _db = db; - // busy_timeout is per-connection, not persisted in the DB file — set it on - // every open. Without it, two processes opening the same SQLite file at - // once (e.g. the daemon's flush and a manual `smriti ingest --force`) fail - // immediately with "database is locked" instead of retrying briefly. - db.exec("PRAGMA busy_timeout = 5000"); + // busy_timeout and WAL are set by QMD's openDatabase() inside createStore(), + // which is the only place that runs *before* the cold-open schema work + // (WAL migration, FTS trigger DDL). Setting them here instead was too late: + // that DDL raced with busy_timeout still at bun:sqlite's default of 0, so a + // daemon flush overlapping a foreground command failed on contact rather + // than queueing. Override with QMD_SQLITE_BUSY_TIMEOUT (ms) if needed. initializeMemoryTables(db as any); initializeSmritiTables(db); seedDefaults(db); From ed5509dcebf53cdaf8f08fc289880a3d7aa6a927 Mon Sep 17 00:00:00 2001 From: Ashutosh Tripathi Date: Tue, 25 Aug 2026 12:47:23 +0530 Subject: [PATCH 2/3] fix(search): tokenize FTS queries instead of interpolating them raw searchFiltered pushed the user's query straight into the FTS5 MATCH expression, so punctuation was parsed as FTS5 grammar rather than as text. 'smriti search node-llama-cpp' failed with 'no such column: llama' and 'smriti search qmd 2.8.3' with 'fts5: syntax error near "."'. Hyphenated names are everywhere in these transcripts, so this hit ordinary queries, not edge cases. buildMemoryFTS5Query had the matching defect on the recall path: it stripped punctuation rather than splitting on it, collapsing '2026.4.10' to '2026410' while the porter unicode61 tokenizer had indexed '2026', '4' and '10'. Silent zero-hit rather than an error. Both now split on the same boundaries the tokenizer uses and AND the parts, so a query tokenizes the way the indexed text did. The builder is exported and shared, and searchFiltered parenthesises the group so the column filter binds to all of it. Same class as QMD upstream #563, which does not reach Smriti because these are Smriti's own query builders. Claude-Session: https://claude.ai/code/session_01HWzCvpW1yLZD3SJTXEojvR --- src/memory.ts | 22 ++++++++++++++-------- src/search/index.ts | 11 +++++++++-- test/search.test.ts | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/memory.ts b/src/memory.ts index f5045c1..6f6a4e3 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -147,17 +147,23 @@ export function initializeMemoryTables(db: Database): void { // FTS5 Query Building (same pattern as store.ts buildFTS5Query) // ============================================================================= -function sanitizeMemoryFTSTerm(term: string): string { - return term.replace(/[^\p{L}\p{N}']/gu, "").toLowerCase(); -} - -function buildMemoryFTS5Query(query: string): string | null { +/** + * Build an FTS5 MATCH expression from a user query. + * + * Splits on everything the `porter unicode61` tokenizer treats as a boundary, + * so a query tokenizes the same way the indexed text did. Without this, + * punctuation either matched nothing ("2026.4.10" collapsed to "2026410", but + * the index holds "2026"/"4"/"10") or leaked into FTS5's own grammar + * ("node-llama-cpp" parsed "llama" as a column name). + * + * Exported so `searchFiltered` builds its MATCH the same way. + */ +export function buildMemoryFTS5Query(query: string): string | null { const terms = query - .split(/\s+/) - .map((t) => sanitizeMemoryFTSTerm(t)) + .split(/[^\p{L}\p{N}']+/u) + .map((t) => t.toLowerCase()) .filter((t) => t.length > 0); if (terms.length === 0) return null; - if (terms.length === 1) return `"${terms[0]}"*`; return terms.map((t) => `"${t}"*`).join(" AND "); } diff --git a/src/search/index.ts b/src/search/index.ts index 2021fa3..3b5eeaf 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -8,6 +8,7 @@ import type { Database } from "bun:sqlite"; import { DEFAULT_SEARCH_LIMIT } from "../config"; import { searchMemoryFTS, searchMemoryVec } from "../qmd"; +import { buildMemoryFTS5Query } from "../memory"; // ============================================================================= // Types @@ -74,9 +75,15 @@ export function searchFiltered( const conditions: string[] = []; const params: any[] = []; - // FTS match condition with column filter + // FTS match condition with column filter. The query is tokenized the same + // way the index was (see buildMemoryFTS5Query) rather than interpolated raw + // — raw user input was parsed as FTS5 grammar, so "node-llama-cpp" failed + // with `no such column: llama` and "2.8.3" with `syntax error near "."`. + // Parenthesised so the column filter binds to the whole AND group. + const ftsQuery = buildMemoryFTS5Query(query); + if (!ftsQuery) return []; conditions.push(`memory_fts MATCH ?`); - params.push(`{${columns.join(" ")}} : ${query}`); + params.push(`{${columns.join(" ")}} : (${ftsQuery})`); // Category filter if (filters.category) { diff --git a/test/search.test.ts b/test/search.test.ts index 18d2334..7768e3e 100644 --- a/test/search.test.ts +++ b/test/search.test.ts @@ -13,6 +13,7 @@ import { insertVoiceNote, } from "../src/db"; import { searchFiltered, listSessions } from "../src/search/index"; +import { buildMemoryFTS5Query } from "../src/memory"; let db: Database; @@ -77,7 +78,8 @@ beforeAll(() => { ('s3', 'user', 'The login page has an error when submitting', 'h5', '${now}'), ('s3', 'assistant', 'Fixed the login bug by validating input', 'h6', '${now}'), ('s4', 'user', 'Help me build a tax calculator', 'h7', '${now}'), - ('s4', 'assistant', 'Here is a tax calculator implementation', 'h8', '${now}'); + ('s4', 'assistant', 'Here is a tax calculator implementation', 'h8', '${now}'), + ('s1', 'user', 'Bumped qmd to 2026.4.10 using node-llama-cpp and sqlite-vec', 'h9', '${now}'); `); // Insert sidecar content for s4 (claude-web session) @@ -245,3 +247,33 @@ test("migrateFTSToV2 is idempotent", () => { const results = searchFiltered(db, "calculateTax"); expect(results.length).toBeGreaterThan(0); }); + +// Regression: user queries were interpolated into the FTS5 MATCH expression +// verbatim, so punctuation was parsed as FTS5 grammar rather than tokenized. +// "node-llama-cpp" raised `no such column: llama` and "2026.4.10" raised +// `fts5: syntax error near "."`. Mirrors QMD upstream #563. + +test("searchFiltered matches hyphenated terms instead of erroring", () => { + const results = searchFiltered(db, "node-llama-cpp", { limit: 10 }); + expect(results.length).toBeGreaterThan(0); + expect(results.some((r) => r.session_id === "s1")).toBe(true); +}); + +test("searchFiltered matches dotted version strings", () => { + const results = searchFiltered(db, "2026.4.10", { limit: 10 }); + expect(results.length).toBeGreaterThan(0); + expect(results.some((r) => r.session_id === "s1")).toBe(true); +}); + +test("searchFiltered still narrows — punctuated queries are ANDed, not dropped", () => { + expect(searchFiltered(db, "sqlite-vec", { limit: 10 }).length).toBeGreaterThan(0); + expect(searchFiltered(db, "nothing-here-xyz", { limit: 10 }).length).toBe(0); +}); + +test("buildMemoryFTS5Query tokenizes on the same boundaries as the index", () => { + expect(buildMemoryFTS5Query("node-llama-cpp")).toBe('"node"* AND "llama"* AND "cpp"*'); + expect(buildMemoryFTS5Query("2026.4.10")).toBe('"2026"* AND "4"* AND "10"*'); + expect(buildMemoryFTS5Query("plain")).toBe('"plain"*'); + expect(buildMemoryFTS5Query(" ")).toBe(null); + expect(buildMemoryFTS5Query("!!!")).toBe(null); +}); From 8c1c64192e70914e0332639784e1b592a887341b Mon Sep 17 00:00:00 2001 From: Ashutosh Tripathi Date: Tue, 25 Aug 2026 12:47:58 +0530 Subject: [PATCH 3/3] chore(release): 0.9.2 v0.9.1 was already tagged on 2026-08-02 (pointing at eaf82b8), so the next stable version is 0.9.2. Also realigns package.json, which the auto-release job does not bump and which still read 0.9.0 at v0.9.1. Claude-Session: https://claude.ai/code/session_01HWzCvpW1yLZD3SJTXEojvR --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d6b64d..365b7e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,48 @@ +## [0.9.2] - 2026-08-25 + +### 🎯 Release Overview +Bug-fix release. Two search failures that hit ordinary queries, and a +concurrency crash between the daemon and foreground commands. No new features, +no CLI changes, no migration required. + +### 🐛 Fixes + +#### Search no longer errors on punctuated queries +- `smriti search node-llama-cpp` failed with `no such column: llama`; + `smriti search "qmd 2.8.3"` failed with `fts5: syntax error near "."`. + `searchFiltered` interpolated the raw query into the FTS5 MATCH expression, + so punctuation was parsed as FTS5 grammar rather than as text. Hyphenated + names are common in these transcripts, so this hit everyday searches. +- The recall path had the matching defect more quietly: `buildMemoryFTS5Query` + stripped punctuation instead of splitting on it, collapsing `2026.4.10` to + `2026410` while the index held `2026`, `4`, `10`. Silent zero results. +- Both paths now split on the boundaries the `porter unicode61` tokenizer + uses and AND the parts, so a query tokenizes the way the indexed text did. + +#### Daemon no longer races foreground commands into "database is locked" +- `createStore()` runs the WAL migration and FTS trigger DDL before returning, + so the `busy_timeout` set on the returned handle arrived too late and that + DDL raced at bun:sqlite's default of 0. The daemon re-runs `createStore()` + per flush, so any overlap with a foreground `recall` / `embed` / `ingest` + failed on contact. +- QMD's `openDatabase()` now owns both `busy_timeout` (120s default, + `QMD_SQLITE_BUSY_TIMEOUT` to override) and a retrying WAL migration. +- Measured: six concurrent cold opens on one fresh DB went from **1/6** + succeeding to **6/6**. + +### 🔧 Dependencies +- QMD submodule synced to upstream `v2.8.3` (155 commits, from `v2.1.0`). + Zero breaking changes to the APIs Smriti consumes. `content_vectors` gains + an `embed_fingerprint` column, populated automatically for new vectors; + existing embeddings are unaffected and no re-embed is needed. +- QMD's embedding runtime moves to node-llama-cpp 3.20.0 (llama.cpp b10361). + +### 📝 Notes +- `CHANGELOG.md` has no entries for 0.7.0 through 0.9.1; those releases were + tagged by the auto-release job but never written up here. `package.json` also + drifted — it still read `0.9.0` at the `v0.9.1` tag, because auto-release + tags without bumping the manifest. This release realigns them. + ## [0.6.0] - 2026-03-14 ### 🎯 Release Overview diff --git a/package.json b/package.json index ef2b85c..8593d08 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "smriti", - "version": "0.9.0", + "version": "0.9.2", "description": "Smriti - Unified memory layer across all AI agents", "type": "module", "bin": {