fix: FTS query tokenization and daemon/foreground DB contention (v0.9.1)

CHANGELOG.md

@@ -1,3 +1,48 @@
+## [0.9.2] - 2026-08-25
+
+### 🎯 Release Overview
+Bug-fix release. Two search failures that hit ordinary queries, and a
+concurrency crash between the daemon and foreground commands. No new features,
+no CLI changes, no migration required.
+
+### 🐛 Fixes
+
+#### Search no longer errors on punctuated queries
+- `smriti search node-llama-cpp` failed with `no such column: llama`;
+  `smriti search "qmd 2.8.3"` failed with `fts5: syntax error near "."`.
+  `searchFiltered` interpolated the raw query into the FTS5 MATCH expression,
+  so punctuation was parsed as FTS5 grammar rather than as text. Hyphenated
+  names are common in these transcripts, so this hit everyday searches.
+- The recall path had the matching defect more quietly: `buildMemoryFTS5Query`
+  stripped punctuation instead of splitting on it, collapsing `2026.4.10` to
+  `2026410` while the index held `2026`, `4`, `10`. Silent zero results.
+- Both paths now split on the boundaries the `porter unicode61` tokenizer
+  uses and AND the parts, so a query tokenizes the way the indexed text did.
+
+#### Daemon no longer races foreground commands into "database is locked"
+- `createStore()` runs the WAL migration and FTS trigger DDL before returning,
+  so the `busy_timeout` set on the returned handle arrived too late and that
+  DDL raced at bun:sqlite's default of 0. The daemon re-runs `createStore()`
+  per flush, so any overlap with a foreground `recall` / `embed` / `ingest`
+  failed on contact.
+- QMD's `openDatabase()` now owns both `busy_timeout` (120s default,
+  `QMD_SQLITE_BUSY_TIMEOUT` to override) and a retrying WAL migration.
+- Measured: six concurrent cold opens on one fresh DB went from **1/6**
+  succeeding to **6/6**.
+
+### 🔧 Dependencies
+- QMD submodule synced to upstream `v2.8.3` (155 commits, from `v2.1.0`).
+  Zero breaking changes to the APIs Smriti consumes. `content_vectors` gains
+  an `embed_fingerprint` column, populated automatically for new vectors;
+  existing embeddings are unaffected and no re-embed is needed.
+- QMD's embedding runtime moves to node-llama-cpp 3.20.0 (llama.cpp b10361).
+
+### 📝 Notes
+- `CHANGELOG.md` has no entries for 0.7.0 through 0.9.1; those releases were
+  tagged by the auto-release job but never written up here. `package.json` also
+  drifted — it still read `0.9.0` at the `v0.9.1` tag, because auto-release
+  tags without bumping the manifest. This release realigns them.
+
 ## [0.6.0] - 2026-03-14
 
 ### 🎯 Release Overview

package.json

@@ -1,6 +1,6 @@
 {
   "name": "smriti",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "Smriti - Unified memory layer across all AI agents",
   "type": "module",
   "bin": {

src/db.ts

@@ -730,11 +730,12 @@ export async function initSmriti(dbPath?: string): Promise<Database> {
   setQmdStore(store);
   const db = store.internal.db as unknown as Database;
   _db = db;
-  // busy_timeout is per-connection, not persisted in the DB file — set it on
-  // every open. Without it, two processes opening the same SQLite file at
-  // once (e.g. the daemon's flush and a manual `smriti ingest --force`) fail
-  // immediately with "database is locked" instead of retrying briefly.
-  db.exec("PRAGMA busy_timeout = 5000");
+  // busy_timeout and WAL are set by QMD's openDatabase() inside createStore(),
+  // which is the only place that runs *before* the cold-open schema work
+  // (WAL migration, FTS trigger DDL). Setting them here instead was too late:
+  // that DDL raced with busy_timeout still at bun:sqlite's default of 0, so a
+  // daemon flush overlapping a foreground command failed on contact rather
+  // than queueing. Override with QMD_SQLITE_BUSY_TIMEOUT (ms) if needed.
   initializeMemoryTables(db as any);
   initializeSmritiTables(db);
   seedDefaults(db);

src/memory.ts

@@ -147,17 +147,23 @@ export function initializeMemoryTables(db: Database): void {
 // FTS5 Query Building (same pattern as store.ts buildFTS5Query)
 // =============================================================================
 
-function sanitizeMemoryFTSTerm(term: string): string {
-  return term.replace(/[^\p{L}\p{N}']/gu, "").toLowerCase();
-}
-
-function buildMemoryFTS5Query(query: string): string | null {
+/**
+ * Build an FTS5 MATCH expression from a user query.
+ *
+ * Splits on everything the `porter unicode61` tokenizer treats as a boundary,
+ * so a query tokenizes the same way the indexed text did. Without this,
+ * punctuation either matched nothing ("2026.4.10" collapsed to "2026410", but
+ * the index holds "2026"/"4"/"10") or leaked into FTS5's own grammar
+ * ("node-llama-cpp" parsed "llama" as a column name).
+ *
+ * Exported so `searchFiltered` builds its MATCH the same way.
+ */
+export function buildMemoryFTS5Query(query: string): string | null {
   const terms = query
-    .split(/\s+/)
-    .map((t) => sanitizeMemoryFTSTerm(t))
+    .split(/[^\p{L}\p{N}']+/u)
+    .map((t) => t.toLowerCase())
     .filter((t) => t.length > 0);
   if (terms.length === 0) return null;
-  if (terms.length === 1) return `"${terms[0]}"*`;
   return terms.map((t) => `"${t}"*`).join(" AND ");
 }
 

src/search/index.ts

@@ -8,6 +8,7 @@
 import type { Database } from "bun:sqlite";
 import { DEFAULT_SEARCH_LIMIT } from "../config";
 import { searchMemoryFTS, searchMemoryVec } from "../qmd";
+import { buildMemoryFTS5Query } from "../memory";
 
 // =============================================================================
 // Types
@@ -74,9 +75,15 @@ export function searchFiltered(
   const conditions: string[] = [];
   const params: any[] = [];
 
-  // FTS match condition with column filter
+  // FTS match condition with column filter. The query is tokenized the same
+  // way the index was (see buildMemoryFTS5Query) rather than interpolated raw
+  // — raw user input was parsed as FTS5 grammar, so "node-llama-cpp" failed
+  // with `no such column: llama` and "2.8.3" with `syntax error near "."`.
+  // Parenthesised so the column filter binds to the whole AND group.
+  const ftsQuery = buildMemoryFTS5Query(query);
+  if (!ftsQuery) return [];
   conditions.push(`memory_fts MATCH ?`);
-  params.push(`{${columns.join(" ")}} : ${query}`);
+  params.push(`{${columns.join(" ")}} : (${ftsQuery})`);
 
   // Category filter
   if (filters.category) {

test/search.test.ts

@@ -13,6 +13,7 @@ import {
   insertVoiceNote,
 } from "../src/db";
 import { searchFiltered, listSessions } from "../src/search/index";
+import { buildMemoryFTS5Query } from "../src/memory";
 
 let db: Database;
 
@@ -77,7 +78,8 @@ beforeAll(() => {
       ('s3', 'user', 'The login page has an error when submitting', 'h5', '${now}'),
       ('s3', 'assistant', 'Fixed the login bug by validating input', 'h6', '${now}'),
       ('s4', 'user', 'Help me build a tax calculator', 'h7', '${now}'),
-      ('s4', 'assistant', 'Here is a tax calculator implementation', 'h8', '${now}');
+      ('s4', 'assistant', 'Here is a tax calculator implementation', 'h8', '${now}'),
+      ('s1', 'user', 'Bumped qmd to 2026.4.10 using node-llama-cpp and sqlite-vec', 'h9', '${now}');
   `);
 
   // Insert sidecar content for s4 (claude-web session)
@@ -245,3 +247,33 @@ test("migrateFTSToV2 is idempotent", () => {
   const results = searchFiltered(db, "calculateTax");
   expect(results.length).toBeGreaterThan(0);
 });
+
+// Regression: user queries were interpolated into the FTS5 MATCH expression
+// verbatim, so punctuation was parsed as FTS5 grammar rather than tokenized.
+// "node-llama-cpp" raised `no such column: llama` and "2026.4.10" raised
+// `fts5: syntax error near "."`. Mirrors QMD upstream #563.
+
+test("searchFiltered matches hyphenated terms instead of erroring", () => {
+  const results = searchFiltered(db, "node-llama-cpp", { limit: 10 });
+  expect(results.length).toBeGreaterThan(0);
+  expect(results.some((r) => r.session_id === "s1")).toBe(true);
+});
+
+test("searchFiltered matches dotted version strings", () => {
+  const results = searchFiltered(db, "2026.4.10", { limit: 10 });
+  expect(results.length).toBeGreaterThan(0);
+  expect(results.some((r) => r.session_id === "s1")).toBe(true);
+});
+
+test("searchFiltered still narrows — punctuated queries are ANDed, not dropped", () => {
+  expect(searchFiltered(db, "sqlite-vec", { limit: 10 }).length).toBeGreaterThan(0);
+  expect(searchFiltered(db, "nothing-here-xyz", { limit: 10 }).length).toBe(0);
+});
+
+test("buildMemoryFTS5Query tokenizes on the same boundaries as the index", () => {
+  expect(buildMemoryFTS5Query("node-llama-cpp")).toBe('"node"* AND "llama"* AND "cpp"*');
+  expect(buildMemoryFTS5Query("2026.4.10")).toBe('"2026"* AND "4"* AND "10"*');
+  expect(buildMemoryFTS5Query("plain")).toBe('"plain"*');
+  expect(buildMemoryFTS5Query("   ")).toBe(null);
+  expect(buildMemoryFTS5Query("!!!")).toBe(null);
+});