fix(core): fill a filtered vector window instead of stopping at rejected neighbours

CHANGELOG.md

@@ -23,6 +23,28 @@
   database is skipped with a warning and an inexact total, and a retryable outage
   or a server too old to attribute its results fails the whole page.
 
+- **#1558**: Vector retrieval reads only the projects in scope and fills the window it
+  asks for. The sqlite-vec table gains a `project_id` partition key, so a scoped
+  nearest-neighbour query ranks each project's own vectors instead of taking the k
+  nearest across the whole database and discarding the out-of-scope ones, which
+  could leave a small project with an empty page for a query its notes answered.
+  Existing local storage is carried into the partitioned table without re-embedding.
+  On Postgres the nearest-neighbour statement now runs on the HNSW index (its
+  tie-break sort keys had kept the planner on an exact scan of every vector), with
+  `hnsw.ef_search` sized to the candidate window and an iterative scan that keeps
+  going until the scope and manifest filters have admitted enough rows. That scan
+  needs pgvector 0.8 or later; an older extension is reported as a dependency error
+  instead of quietly returning short windows.
+
+- **#1558**: A vector or hybrid search with structured filters (note types, dates,
+  categories, metadata, path prefixes, valid time) now fills its candidate window.
+  The vector index ranks by similarity alone, so a window taken straight from it and
+  filtered afterwards could hold few admitted rows while more sat just past it, and
+  the page came back short although matches existed. The reader re-reads the window
+  with a bounded geometric overfetch until it holds enough admitted rows, the ranking
+  is exhausted, or its tail falls below the similarity threshold. Unfiltered searches
+  read their window once, as before.
+
 - **#1512**: Word, PowerPoint, and CSV files get the same sidecar Markdown note a
   PDF gets. `bm import document <path>` indexes the project, extracts the file,
   and writes `<file>.<ext>.md` next to it plus a run note under

src/basic_memory/models/search.py

@@ -223,11 +223,18 @@
 
 
 def create_sqlite_search_vector_embeddings(dimensions: int) -> DDL:
-    """Build sqlite-vec virtual table DDL for the configured embedding dimension."""
+    """Build sqlite-vec virtual table DDL for the configured embedding dimension.
+
+    ``project_id`` is a vec0 partition key: a scoped nearest-neighbour query reads
+    only the partitions in scope, instead of ranking every project's vectors in the
+    database and discarding the out-of-scope ones afterwards, which left a small
+    project with an under-filled window whenever a larger neighbour sat closer.
+    """
     return DDL(
         f"""
 CREATE VIRTUAL TABLE IF NOT EXISTS search_vector_embeddings
 USING vec0(
+    project_id integer partition key,
     embedding float[{dimensions}],
     +source_hash text
 )

src/basic_memory/repository/pgvector_index.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import re
 from collections.abc import Sequence
 
 from loguru import logger
@@ -23,6 +24,32 @@
 )
 
 
+# pgvector's HNSW scan hands back at most ``hnsw.ef_search`` rows, 40 by default,
+# whatever the LIMIT asks for; the server caps the setting at 1000.
+HNSW_EF_SEARCH_DEFAULT = 40
+HNSW_EF_SEARCH_MAX = 1000
+
+_PGVECTOR_VERSION = re.compile(r"^(\d+)\.(\d+)")
+
+
+def pgvector_supports_iterative_scan(extversion: str) -> bool:
+    """Whether this pgvector keeps scanning an HNSW index until the LIMIT is filled.
+
+    Iterative index scans arrived in pgvector 0.8.0. Before that, a scan stops at
+    ``hnsw.ef_search`` candidates, and a filter applied afterwards (the manifest
+    join, a scope narrower than the table) leaves the window under-filled, so the
+    adapter cannot promise the nearest ``limit`` rows in scope. A version string
+    the pattern cannot read counts as older.
+    """
+    match = _PGVECTOR_VERSION.match(extversion)
+    return match is not None and (int(match.group(1)), int(match.group(2))) >= (0, 8)
+
+
+def hnsw_ef_search_for(limit: int) -> int:
+    """The candidate-list size that lets one HNSW scan return ``limit`` rows."""
+    return min(max(limit, HNSW_EF_SEARCH_DEFAULT), HNSW_EF_SEARCH_MAX)
+
+
 class PgVectorIndex:
     """Persist and query semantic vectors in PostgreSQL with pgvector."""
 
@@ -56,6 +83,19 @@ async def initialize(self) -> None:
                     raise SemanticDependenciesMissingError(
                         "pgvector extension is unavailable for this Postgres database."
                     ) from exc
+                version = await session.execute(
+                    text("SELECT extversion FROM pg_extension WHERE extname = 'vector'")
+                )
+                extversion = str(version.scalar_one())
+                # Trigger: the installed pgvector predates iterative index scans.
+                # Why: without them a scoped query silently returns fewer rows than
+                #   the window asked for; a deployment gap should read as one.
+                # Outcome: a typed dependency error the API reports as a bad request.
+                if not pgvector_supports_iterative_scan(extversion):
+                    raise SemanticDependenciesMissingError(
+                        f"pgvector {extversion} predates iterative index scans; semantic "
+                        "search needs pgvector 0.8 or later (ALTER EXTENSION vector UPDATE)."
+                    )
 
                 existing_dimensions = await self._existing_dimensions(session)
                 storage_missing = existing_dimensions is None
@@ -322,10 +362,25 @@ async def search(
         embeddings_in_scope = projects.predicate("e.project_id", params)
         chunks_in_scope = projects.predicate("c.project_id", params)
         async with db.scoped_session(self._session_maker) as session:
+            # Both settings are transaction-local, so they last exactly as long as
+            # this scoped session. The scan is sized to the window it must fill
+            # and continues past that until the scope and manifest filters have
+            # admitted enough rows, instead of stopping at the first ef_search
+            # candidates and returning whatever of them survived.
+            await session.execute(
+                text("SELECT set_config('hnsw.ef_search', :ef_search, true)"),
+                {"ef_search": str(hnsw_ef_search_for(limit))},
+            )
+            await session.execute(
+                text("SELECT set_config('hnsw.iterative_scan', 'relaxed_order', true)")
+            )
+            # A relaxed iterative scan may hand rows back slightly out of distance
+            # order, so the window is taken by distance alone and sorted once more.
             result = await session.execute(
                 text(
+                    "WITH nearest AS MATERIALIZED ("
                     "SELECT c.entity_id, c.chunk_key, "
-                    "1 - (e.embedding <=> CAST(:query AS vector)) AS similarity "
+                    "e.embedding <=> CAST(:query AS vector) AS distance "
                     "FROM search_vector_embeddings e "
                     "JOIN search_vector_chunks c ON c.id = e.chunk_id "
                     f"WHERE {embeddings_in_scope} "
@@ -335,9 +390,12 @@ async def search(
                     "AND c.embedding_status = 'ready' "
                     "AND c.embedding_model = :embedding_identity "
                     "AND e.source_hash = c.source_hash "
-                    "ORDER BY e.embedding <=> CAST(:query AS vector), "
-                    "c.entity_id ASC, c.chunk_key ASC "
+                    "ORDER BY e.embedding <=> CAST(:query AS vector) "
                     "LIMIT :limit"
+                    ") "
+                    "SELECT entity_id, chunk_key, 1 - distance AS similarity "
+                    "FROM nearest "
+                    "ORDER BY distance ASC, entity_id ASC, chunk_key ASC"
                 ),
                 params,
             )