From 072ccb7ed1286f6f2d346515e371e50720b97b4e Mon Sep 17 00:00:00 2001 From: phernandez Date: Tue, 15 Sep 2026 11:36:48 -0500 Subject: [PATCH] refactor(core): compile search filters over an explicit ProjectScope First step of #1558: one scope-parameterized read path shared by the single-project route, the database-scoped route, and MCP search_all_projects. This commit moves the FTS compilation out of the project-bound repositories and makes the project set an explicit value. No query behavior changes. - search_scope.ProjectScope: sorted, unique, positive project IDs built once at the boundary; predicate() renders `IN (:scope_N)` or `1 = 0` for empty. - search_filters: FilterDialect (the two spellings that differ per backend), CompiledFilter (named FROM/WHERE/params/order/score instead of a 5-tuple), and shared_filter_conditions for the filters both backends compile the same way. Removes the duplicated filter blocks the two repositories carried. - sqlite_search_query / postgres_search_query: term preparation, relaxed renderers, syntax-error classifiers, and compile_fts_filter as module functions. The SQLite compiler is pure; the repository passes the entity columns in instead of the compiler opening a session. - note_type_filters / temporal_filters take scope= and match search rows on their full (project_id, ...) identity. - SearchRepositoryBase gains self.scope (a scope of one) and drops the abstract _prepare_search_term hook nothing in the base called. - Tests call the module functions directly; Postgres tests patch the classifier on the repository module where it is bound. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019YW9ysxugGGBCNEGzsxtFV Signed-off-by: phernandez --- CHANGELOG.md | 12 + .../repository/note_type_filters.py | 13 +- .../repository/postgres_search_query.py | 667 ++++++++++++++ .../repository/postgres_search_repository.py | 822 +----------------- src/basic_memory/repository/search_filters.py | 147 ++++ .../repository/search_repository_base.py | 20 +- src/basic_memory/repository/search_scope.py | 61 ++ .../repository/sqlite_search_query.py | 423 +++++++++ .../repository/sqlite_search_repository.py | 661 +------------- .../repository/temporal_filters.py | 19 +- tests/repository/test_hybrid_fusion.py | 4 - .../test_postgres_search_quoted_queries.py | 8 +- .../test_postgres_search_repository.py | 85 +- .../test_postgres_search_repository_unit.py | 5 +- .../test_search_file_path_prefix.py | 2 +- .../test_search_relaxed_rendering.py | 16 +- tests/repository/test_search_repository.py | 166 ++-- tests/repository/test_search_scope.py | 47 + tests/repository/test_semantic_search_base.py | 4 - tests/repository/test_semantic_vector_sync.py | 4 - tests/repository/test_vector_pagination.py | 4 - tests/repository/test_vector_threshold.py | 4 - 22 files changed, 1593 insertions(+), 1601 deletions(-) create mode 100644 src/basic_memory/repository/postgres_search_query.py create mode 100644 src/basic_memory/repository/search_filters.py create mode 100644 src/basic_memory/repository/search_scope.py create mode 100644 src/basic_memory/repository/sqlite_search_query.py create mode 100644 tests/repository/test_search_scope.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 239f084dc..4262bb491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,18 @@ Frontmatter is now classified once, by the parser, as present, absent, or malformed, and only the first two are ever written to. +### Internal + +- **#1558**: Search filter compilation now runs over an explicit `ProjectScope` instead of + a repository-bound `project_id`. FTS term preparation and filter compilation moved out + of the SQLite and Postgres repositories into `sqlite_search_query` and + `postgres_search_query` as pure functions returning a `CompiledFilter`, and the filters + both backends share (scope, permalink, directory, item type, category, note type, + `after_date`, valid time, candidate keys) are compiled once in `search_filters`. The + note-type and valid-time predicates match search rows on their full + `(project_id, ...)` identity. Project repositories call the compilers with a scope of + one; no query behavior changes. First step of the shared single/multi-project reader. + ## v0.23.2 (2026-08-25) diff --git a/src/basic_memory/repository/note_type_filters.py b/src/basic_memory/repository/note_type_filters.py index 9da86df09..9c8767845 100644 --- a/src/basic_memory/repository/note_type_filters.py +++ b/src/basic_memory/repository/note_type_filters.py @@ -33,6 +33,7 @@ from typing import Any, Sequence +from basic_memory.repository.search_scope import ProjectScope from basic_memory.schemas.search import SearchItemType SEARCH_TABLE = "search_index" @@ -49,6 +50,7 @@ def build_note_type_predicate( note_types: Sequence[str], params: dict[str, Any], *, + scope: ProjectScope, note_type_value: str, ) -> str: """Build the WHERE-clause fragment restricting rows to notes of the given types. @@ -57,19 +59,22 @@ def build_note_type_predicate( documented case-insensitive, so both sides are folded to lowercase. Binds are added to `params` in place, following the convention the surrounding FTS - query builders already use. `project_id` is bound by the caller for the whole query. + query builders already use. The owning note is matched on `(project_id, id)`: the + search row's identity is composite, and the subquery is restricted to `scope` so an + owner outside the caller's projects can never admit a row. """ placeholders = [] for index, note_type in enumerate(note_types): name = f"note_type_{index}" params[name] = note_type.lower() placeholders.append(f":{name}") + owner_scope = scope.predicate(f"{_OWNER}.project_id", params) return ( - f"{SEARCH_TABLE}.entity_id IN (\n" - f" SELECT {_OWNER}.id\n" + f"({SEARCH_TABLE}.project_id, {SEARCH_TABLE}.entity_id) IN (\n" + f" SELECT {_OWNER}.project_id, {_OWNER}.id\n" f" FROM {SEARCH_TABLE} AS {_OWNER}\n" f" WHERE {_OWNER}.type = '{SearchItemType.ENTITY.value}'\n" - f" AND {_OWNER}.project_id = :project_id\n" + f" AND {owner_scope}\n" f" AND LOWER({note_type_value}) IN ({', '.join(placeholders)}))" ) diff --git a/src/basic_memory/repository/postgres_search_query.py b/src/basic_memory/repository/postgres_search_query.py new file mode 100644 index 000000000..ea30070b7 --- /dev/null +++ b/src/basic_memory/repository/postgres_search_query.py @@ -0,0 +1,667 @@ +"""PostgreSQL tsquery preparation: term syntax and filter compilation. + +Pure functions over a ``ProjectScope`` and the caller's filters. Nothing here opens +a session or owns an index. +""" + +import json +import re +from collections.abc import Sequence +from datetime import datetime +from typing import Any + +from basic_memory.repository.metadata_filters import parse_metadata_filters +from basic_memory.repository.script_ngrams import analyze_script_query +from basic_memory.repository.search_filters import ( + AFTER_DATE_ORDER_BY, + POSTGRES_FILTER_DIALECT, + CompiledFilter, + shared_filter_conditions, +) +from basic_memory.repository.search_query import relaxation_word_tokens, relaxed_query_words +from basic_memory.repository.search_repository_base import ( + SearchIndexKey, + metadata_contains_like_condition, + metadata_filter_content_type_condition, +) +from basic_memory.repository.search_scope import ProjectScope +from basic_memory.schemas.search import SearchItemType +from basic_memory.temporal import TemporalFilter + +_TSQUERY_OPERAND_PATTERN = re.compile(r"'(?:''|[^'])*'(?::\*)?|[^\s&|!()]+") +_TSQUERY_WORD_PATTERN = re.compile(r"[^\W_]+(?:'[^\W_]+)?", re.UNICODE) +_QUOTED_QUERY_PATTERN = re.compile(r'"([^"]*)"') +_BOOLEAN_WORDS = frozenset({"AND", "OR", "NOT"}) +_TSQUERY_METACHARACTERS = frozenset("&|!:<>") +# tsquery special characters that must not reach the parser as text. +_TSQUERY_SPECIAL_CHARS = ("&", "|", "!", "(", ")", ":") + + +# --- Term preparation --- + + +def _tsquery_operands(processed_text: str) -> list[tuple[str, str]]: + """Return unique (query operand, representative text) pairs in source order.""" + operands: dict[str, str] = {} + for operand in _TSQUERY_OPERAND_PATTERN.findall(processed_text): + representative = operand.removesuffix(":*") + if representative.startswith("'") and representative.endswith("'"): + representative = representative[1:-1].replace("''", "'") + operands.setdefault(operand, representative) + continue + + # An unquoted apostrophe is invalid tsquery syntax. Keep the literal word for + # the synthetic document, but quote and escape its probe so a strict syntax + # failure can proceed to the relaxed retry. + if "'" in representative: + escaped = "'{}'".format(representative.replace("'", "''")) + safe_operand = f"{escaped}:*" if operand.endswith(":*") else escaped + operands.setdefault(safe_operand, representative) + continue + + # PostgreSQL legitimately parses punctuation inside operands such as + # ``v0.13.0b2:*`` and ``auth-service:*``. Preserve those bytes so the + # synthetic document is tokenized the same way as the original note. + if "<" not in representative and ">" not in representative: + operands.setdefault(operand, representative) + continue + + # A malformed strict operand (for example ``foo str: + """Render user text as complete, individually escaped tsquery operands.""" + words = relaxation_word_tokens(text_value) + if drop_boolean_words: + words = [word for word in words if word.upper() not in _BOOLEAN_WORDS] + if not words: + return "NOSPECIALCHARS:*" + + operands: list[str] = [] + for word in words: + escaped_word = "'{}'".format(word.replace("'", "''")) if "'" in word else word + operands.append(f"{escaped_word}:*" if is_prefix else escaped_word) + return operator.join(operands) + + +def _render_boolean_operand(operand: str) -> str: + """Preserve safe structured text while escaping tsquery syntax bytes.""" + if "'" not in operand and not any(char in _TSQUERY_METACHARACTERS for char in operand): + return operand + return _render_tsquery_words(operand, operator=" & ", is_prefix=False) + + +def _has_valid_boolean_shape(expression: str) -> bool: + """Reject incomplete operator structure before it reaches strict ``to_tsquery``.""" + depth = 0 + for char in expression: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth < 0: + return False + if depth: + return False + + stripped = expression.strip() + if not stripped or stripped[0] in "&|" or stripped[-1] in "&|!": + return False + return not any( + re.search(pattern, stripped) + for pattern in ( + r"[&|]\s*[&|]", + r"!\s*[&|)]", + r"\(\s*[&|)]", + r"[&|!(]\s*\)", + ) + ) + + +def prepare_single_term(term: str, is_prefix: bool = True) -> str: + """Prepare one search term with no Boolean operators. + + Multi-word queries become ``word1 & word2``; ``is_prefix`` adds the ``:*`` + suffix; tsquery special characters are removed. + """ + if not term or not term.strip(): + return term + + term = term.strip() + + # An existing wildcard pattern converts to the tsquery prefix operator. + if "*" in term: + return term.replace("*", ":*") + + cleaned_term = term + for char in _TSQUERY_SPECIAL_CHARS: + cleaned_term = cleaned_term.replace(char, " ") + + if " " in cleaned_term: + # Strip sentence punctuation from word edges so question-form queries + # produce clean lexemes (parity with SQLite FTS5 prep). The tsquery tokenizer + # ignores this punctuation anyway; leaving it only risks syntax errors. + words = [w.strip("?!.,;") for w in cleaned_term.split()] + words = [w for w in words if w] + if not words: + # Only special characters remained; emit a term that cannot error. + return "NOSPECIALCHARS:*" + prepared_words = [f"{word}:*" for word in words] if is_prefix else words + return " & ".join(prepared_words) + + # Single word: strip edge punctuation and guard the now-empty case so a bare + # ":*" never reaches tsquery. + cleaned_term = cleaned_term.strip().strip("?!.,;") + if not cleaned_term: + return "NOSPECIALCHARS:*" + if is_prefix: + return f"{cleaned_term}:*" + return cleaned_term + + +def prepare_boolean_query(query: str) -> str: + """Convert a Boolean query to tsquery operators (``&``, ``|``, ``!``). + + Examples: ``coffee AND brewing`` -> ``coffee & brewing``; + ``(pour OR french) AND press`` -> ``(pour | french) & press``; + ``coffee NOT decaf`` -> ``coffee & !decaf``. + """ + # PostgreSQL's strict to_tsquery grammar does not accept web-style double quotes. + # Convert complete quoted groups first so operator-looking words inside them + # remain text and every word becomes a complete operand. + quoted_phrases: dict[str, str] = {} + + def replace_quoted_phrase(match: re.Match[str]) -> str: + phrase = _render_tsquery_words(match.group(1), operator=" & ", is_prefix=False) + placeholder = f"BMQUOTEDPHRASE{len(quoted_phrases)}" + while placeholder in query: + placeholder += "X" + quoted_phrases[placeholder] = f"({phrase})" + # Surround the placeholder so quotes adjacent to plain text become explicit + # operands instead of restoring into ``word(group)``. + return f" {placeholder} " + + result = _QUOTED_QUERY_PATTERN.sub(replace_quoted_phrase, query) + if '"' in result: + # An unmatched quote is user text, not a reason to abort the database + # transaction. Boolean-looking words lose their operator role here. + return _render_tsquery_words(query, operator=" & ", is_prefix=True, drop_boolean_words=True) + + # Boolean syntax is the only structure retained from user input. A + # whitespace-delimited operand still needs explicit conjunctions, but PostgreSQL + # must tokenize structured single operands such as ``auth-service`` and + # ``config.json`` exactly as it did before quoted query normalization. + normalized_parts: list[str] = [] + operator_pattern = r"((? str: + """Prepare user text as a tsquery. + + Boolean operators convert to tsquery form, prefix matching uses ``:*``, and + terms are sanitized so they cannot raise tsquery syntax errors. + """ + if '"' in term or any(op in f" {term} " for op in (" AND ", " OR ", " NOT ")): + return prepare_boolean_query(term) + return prepare_single_term(term, is_prefix) + + +def _relaxed_tsquery_term(word: str) -> str: + """Render one relaxed word as a tsquery-safe prefix expression. + + Mirrors the SQLite renderer: a word token can contain an apostrophe, and tsquery + reads that as lexeme-quoting syntax rather than text. Quoting the lexeme and + doubling any interior quote keeps it literal. + """ + if "'" in word: + return "'{}':*".format(word.replace("'", "''")) + return f"{word}:*" + + +def relaxed_tsquery_text(search_text: str | None) -> str | None: + """OR-relaxed tsquery expression for a failed strict query, or None.""" + words = relaxed_query_words(search_text) + if not words: + return None + return " | ".join(_relaxed_tsquery_term(word) for word in words) + + +def is_tsquery_syntax_error(exc: Exception) -> bool: + msg = str(exc).lower() + return ( + "syntax error in tsquery" in msg + or "invalid input syntax for type tsquery" in msg + or "no operand in tsquery" in msg + or "no operator in tsquery" in msg + ) + + +# --- Filter compilation --- + + +def document_fts_vector_sql(processed_texts: Sequence[str], params: dict[str, Any]) -> str: + """Build a query-sized vector representing lexemes found anywhere in one item.""" + operands: dict[str, str] = {} + for processed_text in processed_texts: + for operand, representative in _tsquery_operands(processed_text): + operands.setdefault(operand, representative) + + present_lexemes: list[str] = [] + for index, (operand, representative) in enumerate(operands.items()): + operand_param = f"text_operand_{index}" + representative_param = f"text_representative_{index}" + params[operand_param] = operand + params[representative_param] = representative + present_lexemes.append( + "CASE WHEN (search_index.textsearchable_index_col " + f"@@ to_tsquery('english', :{operand_param}) OR EXISTS (" + "SELECT 1 FROM search_index_fts_chunks AS operand_chunk " + "WHERE operand_chunk.project_id = search_index.project_id " + "AND operand_chunk.search_index_id = search_index.id " + "AND operand_chunk.search_index_type = search_index.type " + "AND operand_chunk.textsearchable_index_col " + f"@@ to_tsquery('english', :{operand_param}))) " + f"THEN :{representative_param} ELSE '' END" + ) + + if not present_lexemes: + return "search_index.textsearchable_index_col" + + # The synthesized text contains at most the query operands, never the note body. + # This preserves document-wide Boolean semantics without recreating an + # unbounded vector. + lexeme_array = f"ARRAY[{', '.join(present_lexemes)}]" + return f"to_tsvector('english', array_to_string({lexeme_array}, ' '))" + + +def _word_candidate_from_clause(scope: ProjectScope, params: dict[str, Any]) -> str: + """Join search rows to the GIN-indexed candidates for the word channel. + + Trigger: PostgreSQL can extract a required-positive query tree. + Why: OR-ing its operands is a safe indexed superset even when terms live in + different chunks. Pure or optional negation returns ``T`` and must retain all + scoped rows for correct semantics. + Outcome: ordinary and required-positive NOT queries use both GIN indexes; only + genuinely unindexable negation scans the scope. + """ + parent_scope = scope.predicate("candidate_parent.project_id", params) + chunk_scope = scope.predicate("candidate_chunk.project_id", params) + all_scope = scope.predicate("candidate_all.project_id", params) + return f""" + search_index JOIN ( + SELECT + candidate_parent.project_id, + candidate_parent.id, + candidate_parent.type + FROM search_index AS candidate_parent + WHERE {parent_scope} + AND querytree(to_tsquery('english', :text)) <> 'T' + AND candidate_parent.textsearchable_index_col + @@ to_tsquery('english', :text_candidate) + UNION + SELECT + candidate_chunk.project_id, + candidate_chunk.search_index_id AS id, + candidate_chunk.search_index_type AS type + FROM search_index_fts_chunks AS candidate_chunk + WHERE {chunk_scope} + AND querytree(to_tsquery('english', :text)) <> 'T' + AND candidate_chunk.textsearchable_index_col + @@ to_tsquery('english', :text_candidate) + UNION + SELECT + candidate_all.project_id, + candidate_all.id, + candidate_all.type + FROM search_index AS candidate_all + WHERE {all_scope} + AND querytree(to_tsquery('english', :text)) = 'T' + ) AS fts_candidate + ON fts_candidate.project_id = search_index.project_id + AND fts_candidate.id = search_index.id + AND fts_candidate.type = search_index.type + """ + + +def _script_candidate_from_clause(scope: ProjectScope, params: dict[str, Any]) -> str: + """Join search rows to the GIN-indexed candidates for the script channel. + + Trigger: a query contains script grams, with or without word terms. + Why: every script phrase is required, while an English word clause can reduce + to an empty tsquery after dictionary processing. + Outcome: start from the parent and child script GIN indexes, then apply every + word and script predicate. + """ + parent_scope = scope.predicate("script_parent.project_id", params) + chunk_scope = scope.predicate("script_candidate.project_id", params) + return f""" + search_index JOIN ( + SELECT + script_parent.project_id, + script_parent.id, + script_parent.type + FROM search_index AS script_parent + WHERE {parent_scope} + AND script_parent.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + UNION + SELECT + script_candidate.project_id, + script_candidate.search_index_id AS id, + script_candidate.search_index_type AS type + FROM search_index_fts_chunks AS script_candidate + WHERE {chunk_scope} + AND script_candidate.script_ngrams_index_col + @@ to_tsquery('simple', :script_candidate_text) + ) AS fts_candidate + ON fts_candidate.project_id = search_index.project_id + AND fts_candidate.id = search_index.id + AND fts_candidate.type = search_index.type + """ + + +def compile_fts_filter( + scope: ProjectScope, + *, + search_text: str | None = None, + permalink: str | None = None, + permalink_match: str | None = None, + title: str | None = None, + note_types: Sequence[str] | None = None, + after_date: datetime | None = None, + search_item_types: Sequence[SearchItemType] | None = None, + categories: Sequence[str] | None = None, + metadata_filters: dict[str, Any] | None = None, + file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, + allow_relaxed: bool = False, + candidate_keys: Sequence[SearchIndexKey] | None = None, +) -> CompiledFilter: + """Compile Postgres FTS FROM/WHERE/score shared by search and count. + + ``allow_relaxed`` widens the indexed candidate set to the relaxed query's + operands too, so the strict statement and its relaxed retry read the same rows. + """ + params: dict[str, Any] = {} + conditions = shared_filter_conditions( + scope, + params, + dialect=POSTGRES_FILTER_DIALECT, + permalink=permalink, + file_path_prefix=file_path_prefix, + candidate_keys=candidate_keys, + search_item_types=search_item_types, + categories=categories, + note_types=note_types, + after_date=after_date, + temporal=temporal, + ) + from_clause = "search_index" + document_vector: str | None = None + script_tsqueries: list[str] = [] + + # Wildcard-only and blank text add no text condition: every row matches. + if search_text and search_text.strip() not in ("", "*"): + script_query = analyze_script_query(search_text.strip()) + if script_query.word_text: + processed_text = prepare_search_term(script_query.word_text) + params["text"] = processed_text + probe_texts = [processed_text] + if allow_relaxed: + relaxed_text = relaxed_tsquery_text(script_query.word_text) + if relaxed_text: + probe_texts.append(relaxed_text) + + candidate_operands: dict[str, None] = {} + for probe_text in probe_texts: + for operand, _representative in _tsquery_operands(probe_text): + candidate_operands.setdefault(operand, None) + if candidate_operands: + params["text_candidate"] = " | ".join(candidate_operands) + from_clause = _word_candidate_from_clause(scope, params) + document_vector = document_fts_vector_sql(probe_texts, params) + word_condition = f"{document_vector} @@ to_tsquery('english', :text)" + if script_query.gram_phrases: + # Trigger: PostgreSQL's English dictionary removes every word term. + # Why: an empty word query must not suppress a required script match. + # Outcome: only mixed queries treat the empty word channel as neutral; + # word-only stopword queries retain their established empty result. + word_condition = f"(numnode(to_tsquery('english', :text)) = 0 OR {word_condition})" + conditions.append(word_condition) + + if script_query.gram_phrases: + script_tsqueries = [ + " <-> ".join(f"'{gram}'" for gram in phrase) for phrase in script_query.gram_phrases + ] + for index, script_tsquery in enumerate(script_tsqueries): + params[f"script_text_{index}"] = script_tsquery + params["script_candidate_text"] = " | ".join( + f"({script_tsquery})" for script_tsquery in script_tsqueries + ) + from_clause = _script_candidate_from_clause(scope, params) + conditions.extend( + "(search_index.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index}) OR EXISTS (" + "SELECT 1 FROM search_index_fts_chunks AS script_chunk " + "WHERE script_chunk.project_id = search_index.project_id " + "AND script_chunk.search_index_id = search_index.id " + "AND script_chunk.search_index_type = search_index.type " + "AND script_chunk.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index})))" + for index in range(len(script_tsqueries)) + ) + + if title: + params["title_text"] = prepare_search_term(title.strip(), is_prefix=False) + conditions.append( + "to_tsvector('english', search_index.title) @@ to_tsquery('english', :title_text)" + ) + + if permalink_match: + permalink_text = permalink_match.lower().strip() + if "*" in permalink_match: + # ``*`` becomes the LIKE wildcard. + params["permalink"] = permalink_text.replace("*", "%") + conditions.append("search_index.permalink LIKE :permalink") + else: + params["permalink"] = permalink_text + conditions.append("search_index.permalink = :permalink") + + # Structured metadata filters use jsonb_extract_path_text() / jsonb_extract_path() + # with parameterized path parts instead of #>> / #> with interpolated paths. + if metadata_filters: + parsed_filters = parse_metadata_filters(metadata_filters) + from_clause = f"{from_clause} JOIN entity ON search_index.entity_id = entity.id" + # Frontmatter filters answer for notes only; see + # metadata_filter_content_type_condition for why every regular file would + # otherwise satisfy a null predicate. + conditions.append(metadata_filter_content_type_condition(params)) + metadata_expr = "entity.entity_metadata::jsonb" + + for idx, filt in enumerate(parsed_filters): + path_param_names: list[str] = [] + for j, part in enumerate(filt.path_parts): + path_param = f"meta_path_{idx}_{j}" + params[path_param] = part + path_param_names.append(f":{path_param}") + path_args = ", ".join(path_param_names) + text_expr = f"jsonb_extract_path_text({metadata_expr}, {path_args})" + json_expr = f"jsonb_extract_path({metadata_expr}, {path_args})" + + # jsonb_extract_path_text returns SQL NULL both for a missing key and for + # an explicit JSON null, the same two cases SQLite's json_extract + # collapses, so the dialects answer ``{"owner": None}`` row for row. + # ``= NULL`` is never true. + if filt.op == "is_null": + conditions.append(f"{text_expr} IS NULL") + continue + + if filt.op == "eq": + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + conditions.append(f"{text_expr} = :{value_param}") + continue + + if filt.op == "in": + placeholders: list[str] = [] + for j, val in enumerate(filt.value): + value_param = f"meta_val_{idx}_{j}" + params[value_param] = val + placeholders.append(f":{value_param}") + conditions.append(f"{text_expr} IN ({', '.join(placeholders)})") + continue + + if filt.op == "contains": + base_param = f"meta_val_{idx}" + tag_conditions: list[str] = [] + # Every requested value must be present. + for j, val in enumerate(filt.value): + tag_param = f"{base_param}_{j}" + params[tag_param] = json.dumps([val]) + # The exact JSONB containment test is the primary path; the + # substring patterns only reach values stored as array text. + like_condition = metadata_contains_like_condition( + text_expr, + val, + param_prefix=tag_param, + params=params, + ) + tag_conditions.append( + f"({json_expr} @> CAST(:{tag_param} AS jsonb) OR {like_condition})" + ) + conditions.append(" AND ".join(tag_conditions)) + continue + + if filt.op in {"gt", "gte", "lt", "lte", "between"}: + compare_expr = ( + f"{text_expr}::double precision" if filt.comparison == "numeric" else text_expr + ) + if filt.op == "between": + min_param = f"meta_val_{idx}_min" + max_param = f"meta_val_{idx}_max" + params[min_param] = filt.value[0] + params[max_param] = filt.value[1] + conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}") + else: + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op] + conditions.append(f"{compare_expr} {operator} :{value_param}") + continue + + # ts_rank per channel. With no text search there is no rank, so the score is 0. + score_parts: list[str] = [] + if document_vector is not None: + score_parts.append( + "GREATEST(" + f"ts_rank({document_vector}, to_tsquery('english', :text)), " + "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text)), " + "COALESCE((SELECT MAX(ts_rank(" + "fts_chunk.textsearchable_index_col, to_tsquery('english', :text))) " + "FROM search_index_fts_chunks AS fts_chunk " + "WHERE fts_chunk.project_id = search_index.project_id " + "AND fts_chunk.search_index_id = search_index.id " + "AND fts_chunk.search_index_type = search_index.type " + "AND fts_chunk.textsearchable_index_col " + "@@ to_tsquery('english', :text)), 0))" + ) + score_parts.extend( + "GREATEST(" + "ts_rank(search_index.script_ngrams_index_col, " + f"to_tsquery('simple', :script_text_{index})), " + "COALESCE((SELECT MAX(ts_rank(script_rank.script_ngrams_index_col, " + f"to_tsquery('simple', :script_text_{index}))) " + "FROM search_index_fts_chunks AS script_rank " + "WHERE script_rank.project_id = search_index.project_id " + "AND script_rank.search_index_id = search_index.id " + "AND script_rank.search_index_type = search_index.type " + "AND script_rank.script_ngrams_index_col " + f"@@ to_tsquery('simple', :script_text_{index})), 0))" + for index in range(len(script_tsqueries)) + ) + # Each condition above is required, so every query component contributes to + # relevance. Taking only the strongest rank would make additional script runs + # invisible. + score_expression = " + ".join(score_parts) if score_parts else "0" + + return CompiledFilter( + from_clause=from_clause, + where_clause=" AND ".join(conditions), + params=params, + order_by_clause=AFTER_DATE_ORDER_BY if after_date else "", + score_expression=score_expression, + ) diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index a6d7e3088..fc89e18da 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -2,7 +2,6 @@ import asyncio import json -import re import time from collections.abc import Sequence from datetime import datetime @@ -21,28 +20,23 @@ from basic_memory.repository.rerank_provider import RerankProvider from basic_memory.repository.rerank_provider_factory import create_rerank_provider from basic_memory.repository.search_index_row import SearchIndexRow -from basic_memory.repository.search_query import relaxed_query_words, relaxation_word_tokens -from basic_memory.repository.script_ngrams import analyze_script_query, build_script_ngrams +from basic_memory.repository.search_query import relaxed_query_words +from basic_memory.repository.script_ngrams import build_script_ngrams from basic_memory.repository.semantic_chunking import VectorChunkRecord from basic_memory.repository.search_repository_base import ( SearchIndexKey, SearchRepositoryBase, VectorChunkState, - candidate_key_restriction_condition, - file_path_prefix_condition, - metadata_contains_like_condition, - metadata_filter_content_type_condition, ) from basic_memory.repository.search_trace import ( SearchTraceCollector, build_fts_page_stage, ) -from basic_memory.repository.metadata_filters import parse_metadata_filters -from basic_memory.repository.note_type_filters import ( - POSTGRES_NOTE_TYPE_VALUE, - build_note_type_predicate, +from basic_memory.repository.postgres_search_query import ( + compile_fts_filter, + is_tsquery_syntax_error, + relaxed_tsquery_text, ) -from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex from basic_memory.repository.semantic_vector_sync import ( @@ -59,111 +53,6 @@ from basic_memory.temporal import TemporalFilter -_TSQUERY_OPERAND_PATTERN = re.compile(r"'(?:''|[^'])*'(?::\*)?|[^\s&|!()]+") -_TSQUERY_WORD_PATTERN = re.compile(r"[^\W_]+(?:'[^\W_]+)?", re.UNICODE) -_QUOTED_QUERY_PATTERN = re.compile(r'"([^"]*)"') -_BOOLEAN_WORDS = frozenset({"AND", "OR", "NOT"}) -_TSQUERY_METACHARACTERS = frozenset("&|!:<>") - - -def _tsquery_operands(processed_text: str) -> list[tuple[str, str]]: - """Return unique (query operand, representative text) pairs in source order.""" - operands: dict[str, str] = {} - for operand in _TSQUERY_OPERAND_PATTERN.findall(processed_text): - representative = operand.removesuffix(":*") - if representative.startswith("'") and representative.endswith("'"): - representative = representative[1:-1].replace("''", "'") - operands.setdefault(operand, representative) - continue - - # An unquoted apostrophe is invalid tsquery syntax. Keep the literal - # word for the synthetic document, but quote and escape its probe so a - # strict syntax failure can proceed to the relaxed retry. - if "'" in representative: - escaped = "'{}'".format(representative.replace("'", "''")) - safe_operand = f"{escaped}:*" if operand.endswith(":*") else escaped - operands.setdefault(safe_operand, representative) - continue - - # PostgreSQL legitimately parses punctuation inside operands such as - # ``v0.13.0b2:*`` and ``auth-service:*``. Preserve those bytes so the - # synthetic document is tokenized the same way as the original note. - if "<" not in representative and ">" not in representative: - operands.setdefault(operand, representative) - continue - - # A malformed strict operand (for example ``foo str: - """Render user text as complete, individually escaped tsquery operands.""" - words = relaxation_word_tokens(text_value) - if drop_boolean_words: - words = [word for word in words if word.upper() not in _BOOLEAN_WORDS] - if not words: - return "NOSPECIALCHARS:*" - - operands = [] - for word in words: - escaped_word = "'{}'".format(word.replace("'", "''")) if "'" in word else word - operands.append(f"{escaped_word}:*" if is_prefix else escaped_word) - return operator.join(operands) - - -def _render_boolean_operand(operand: str) -> str: - """Preserve safe structured text while escaping tsquery syntax bytes.""" - if "'" not in operand and not any(char in _TSQUERY_METACHARACTERS for char in operand): - return operand - return _render_tsquery_words( - operand, - operator=" & ", - is_prefix=False, - ) - - -def _has_valid_boolean_shape(expression: str) -> bool: - """Reject incomplete operator structure before it reaches strict ``to_tsquery``.""" - depth = 0 - for char in expression: - if char == "(": - depth += 1 - elif char == ")": - depth -= 1 - if depth < 0: - return False - if depth: - return False - - stripped = expression.strip() - if not stripped or stripped[0] in "&|" or stripped[-1] in "&|!": - return False - return not any( - re.search(pattern, stripped) - for pattern in ( - r"[&|]\s*[&|]", - r"!\s*[&|)]", - r"\(\s*[&|)]", - r"[&|!(]\s*\)", - ) - ) - - def _strip_nul_from_row(row_data: dict[str, Any]) -> dict[str, Any]: """Strip NUL bytes from all string values in a row dict. @@ -396,236 +285,6 @@ async def _replace_fts_chunks( {"project_id": self.project_id, "chunks": json.dumps(chunks)}, ) - # ------------------------------------------------------------------ - # tsquery preparation (backend-specific) - # ------------------------------------------------------------------ - - @override - def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: - """Prepare a search term for tsquery format. - - Args: - term: The search term to prepare - is_prefix: Whether to add prefix search capability (:* operator) - - Returns: - Formatted search term for tsquery - - For Postgres: - - Boolean operators are converted to tsquery format (&, |, !) - - Prefix matching uses the :* operator - - Terms are sanitized to prevent tsquery syntax errors - """ - # Check for explicit boolean operators - boolean_operators = [" AND ", " OR ", " NOT "] - if '"' in term or any(op in f" {term} " for op in boolean_operators): - return self._prepare_boolean_query(term) - - # For non-Boolean queries, prepare single term - return self._prepare_single_term(term, is_prefix) - - @staticmethod - def _relaxed_tsquery_term(word: str) -> str: - """Render one relaxed word as a tsquery-safe prefix expression. - - Mirrors the SQLite renderer: a word token can contain an apostrophe, and - tsquery reads that as lexeme-quoting syntax rather than text. Quoting the - lexeme and doubling any interior quote keeps it literal. - """ - if "'" in word: - return "'{}':*".format(word.replace("'", "''")) - return f"{word}:*" - - @staticmethod - def _relaxed_tsquery_text(search_text: Optional[str]) -> Optional[str]: - """OR-relaxed tsquery expression for a failed strict query, or None.""" - words = relaxed_query_words(search_text) - if not words: - return None - return " | ".join(PostgresSearchRepository._relaxed_tsquery_term(word) for word in words) - - def _prepare_boolean_query(self, query: str) -> str: - """Convert Boolean query to tsquery format. - - Args: - query: A Boolean query like "coffee AND brewing" or "(pour OR french) AND press" - - Returns: - tsquery-formatted string with & (AND), | (OR), ! (NOT) operators - - Examples: - "coffee AND brewing" -> "coffee & brewing" - "(pour OR french) AND press" -> "(pour | french) & press" - "coffee NOT decaf" -> "coffee & !decaf" - """ - # PostgreSQL's strict to_tsquery grammar does not accept web-style double - # quotes. Convert complete quoted groups first so operator-looking words - # inside them remain text and every word becomes a complete operand. - quoted_phrases: dict[str, str] = {} - - def replace_quoted_phrase(match: re.Match[str]) -> str: - phrase = _render_tsquery_words( - match.group(1), - operator=" & ", - is_prefix=False, - ) - placeholder = f"BMQUOTEDPHRASE{len(quoted_phrases)}" - while placeholder in query: - placeholder += "X" - quoted_phrases[placeholder] = f"({phrase})" - # Surround the placeholder so quotes adjacent to plain text become - # explicit operands instead of restoring into ``word(group)``. - return f" {placeholder} " - - result = _QUOTED_QUERY_PATTERN.sub(replace_quoted_phrase, query) - if '"' in result: - # An unmatched quote is user text, not a reason to abort the database - # transaction. Boolean-looking words lose their operator role here. - return _render_tsquery_words( - query, - operator=" & ", - is_prefix=True, - drop_boolean_words=True, - ) - - # Boolean syntax is the only structure retained from user input. A - # whitespace-delimited operand still needs explicit conjunctions, but - # PostgreSQL must tokenize structured single operands such as - # ``auth-service`` and ``config.json`` exactly as it did before quoted - # query normalization. - normalized_parts: list[str] = [] - operator_pattern = r"((? str: - """Prepare a single search term for tsquery. - - Args: - term: A single search term - is_prefix: Whether to add prefix search capability (:* suffix) - - Returns: - A properly formatted single term for tsquery - - For Postgres tsquery: - - Multi-word queries become "word1 & word2" - - Prefix matching uses ":*" suffix (e.g., "coff:*") - - Special characters that need escaping: & | ! ( ) : - """ - if not term or not term.strip(): - return term - - term = term.strip() - - # Check if term is already a wildcard pattern - if "*" in term: - # Replace * with :* for Postgres prefix matching - return term.replace("*", ":*") - - # Remove tsquery special characters from the search term - # These characters have special meaning in tsquery and cause syntax errors - # if not used as operators - special_chars = ["&", "|", "!", "(", ")", ":"] - cleaned_term = term - for char in special_chars: - cleaned_term = cleaned_term.replace(char, " ") - - # Handle multi-word queries - if " " in cleaned_term: - # Strip sentence punctuation from word edges so question-form - # queries produce clean lexemes (parity with SQLite FTS5 prep). - # The tsquery tokenizer ignores this punctuation anyway; leaving it - # in only risks tsquery syntax errors. Interior characters are kept. - words = [w.strip("?!.,;") for w in cleaned_term.split()] - words = [w for w in words if w] - if not words: - # All characters were special chars, search won't match anything - # Return a safe search term that won't cause syntax errors - return "NOSPECIALCHARS:*" - if is_prefix: - # Add prefix matching to each word - prepared_words = [f"{word}:*" for word in words] - else: - prepared_words = words - # Join with AND operator - return " & ".join(prepared_words) - - # Single word: strip edge punctuation; guard the now-empty case so a - # bare ":*"/"" never reaches tsquery. - cleaned_term = cleaned_term.strip().strip("?!.,;") - if not cleaned_term: - return "NOSPECIALCHARS:*" - if is_prefix: - return f"{cleaned_term}:*" - else: - return cleaned_term - # ------------------------------------------------------------------ # Abstract hook implementations (vector/semantic, Postgres-specific) # ------------------------------------------------------------------ @@ -957,431 +616,6 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non # FTS search (Postgres-specific) # ------------------------------------------------------------------ - @staticmethod - def _is_tsquery_syntax_error(exc: Exception) -> bool: - msg = str(exc).lower() - return ( - "syntax error in tsquery" in msg - or "invalid input syntax for type tsquery" in msg - or "no operand in tsquery" in msg - or "no operator in tsquery" in msg - ) - - async def _build_fts_query_parts( - self, - search_text: Optional[str] = None, - permalink: Optional[str] = None, - permalink_match: Optional[str] = None, - title: Optional[str] = None, - note_types: Optional[List[str]] = None, - after_date: Optional[datetime] = None, - search_item_types: Optional[List[SearchItemType]] = None, - categories: Optional[List[str]] = None, - metadata_filters: Optional[dict[str, Any]] = None, - file_path_prefix: Optional[str] = None, - temporal: Optional[TemporalFilter] = None, - allow_relaxed: bool = False, - candidate_keys: Sequence[SearchIndexKey] | None = None, - ) -> tuple[str, str, dict[str, Any], str, str]: - """Build Postgres FTS FROM/WHERE params shared by search and count.""" - conditions = [] - params = {} - order_by_clause = "" - from_clause = "search_index" - document_vector_sql: str | None = None - script_tsqueries: list[str] = [] - - # Handle text search for title and content using tsvector - if search_text: - if search_text.strip() == "*" or search_text.strip() == "": - # For wildcard searches, don't add any text conditions - pass - else: - script_query = analyze_script_query(search_text.strip()) - if script_query.word_text: - processed_text = self._prepare_search_term(script_query.word_text) - params["text"] = processed_text - probe_texts = [processed_text] - if allow_relaxed: - relaxed_text = self._relaxed_tsquery_text(script_query.word_text) - if relaxed_text: - probe_texts.append(relaxed_text) - - candidate_operands: dict[str, None] = {} - for probe_text in probe_texts: - for operand, _representative in _tsquery_operands(probe_text): - candidate_operands.setdefault(operand, None) - if candidate_operands: - params["text_candidate"] = " | ".join(candidate_operands) - - # Trigger: PostgreSQL can extract a required-positive query tree. - # Why: OR-ing its operands is a safe indexed superset even when - # terms live in different chunks. Pure/optional negation returns - # ``T`` and must retain all project rows for correct semantics. - # Outcome: ordinary and required-positive NOT queries use both - # GIN indexes; only genuinely unindexable negation scans the project. - from_clause = """ - search_index JOIN ( - SELECT - candidate_parent.project_id, - candidate_parent.id, - candidate_parent.type - FROM search_index AS candidate_parent - WHERE candidate_parent.project_id = :project_id - AND querytree(to_tsquery('english', :text)) <> 'T' - AND candidate_parent.textsearchable_index_col - @@ to_tsquery('english', :text_candidate) - UNION - SELECT - candidate_chunk.project_id, - candidate_chunk.search_index_id AS id, - candidate_chunk.search_index_type AS type - FROM search_index_fts_chunks AS candidate_chunk - WHERE candidate_chunk.project_id = :project_id - AND querytree(to_tsquery('english', :text)) <> 'T' - AND candidate_chunk.textsearchable_index_col - @@ to_tsquery('english', :text_candidate) - UNION - SELECT - candidate_all.project_id, - candidate_all.id, - candidate_all.type - FROM search_index AS candidate_all - WHERE candidate_all.project_id = :project_id - AND querytree(to_tsquery('english', :text)) = 'T' - ) AS fts_candidate - ON fts_candidate.project_id = search_index.project_id - AND fts_candidate.id = search_index.id - AND fts_candidate.type = search_index.type - """ - document_vector_sql = self._document_fts_vector_sql(probe_texts, params) - word_condition = f"{document_vector_sql} @@ to_tsquery('english', :text)" - if script_query.gram_phrases: - # Trigger: PostgreSQL's English dictionary removes every word term. - # Why: an empty word query must not suppress a required script match. - # Outcome: only mixed queries treat the empty word channel as neutral; - # word-only stopword queries retain their established empty result. - word_condition = ( - f"(numnode(to_tsquery('english', :text)) = 0 OR {word_condition})" - ) - conditions.append(word_condition) - - if script_query.gram_phrases: - script_tsqueries = [ - " <-> ".join(f"'{gram}'" for gram in phrase) - for phrase in script_query.gram_phrases - ] - for index, script_tsquery in enumerate(script_tsqueries): - params[f"script_text_{index}"] = script_tsquery - # Trigger: a query contains script grams, with or without word terms. - # Why: every script phrase is required, while an English word clause can - # reduce to an empty tsquery after dictionary processing. - # Outcome: start from the parent and child script GIN indexes, then apply - # every word and script predicate below. - params["script_candidate_text"] = " | ".join( - f"({script_tsquery})" for script_tsquery in script_tsqueries - ) - from_clause = """ - search_index JOIN ( - SELECT - script_parent.project_id, - script_parent.id, - script_parent.type - FROM search_index AS script_parent - WHERE script_parent.project_id = :project_id - AND script_parent.script_ngrams_index_col - @@ to_tsquery('simple', :script_candidate_text) - UNION - SELECT - script_candidate.project_id, - script_candidate.search_index_id AS id, - script_candidate.search_index_type AS type - FROM search_index_fts_chunks AS script_candidate - WHERE script_candidate.project_id = :project_id - AND script_candidate.script_ngrams_index_col - @@ to_tsquery('simple', :script_candidate_text) - ) AS fts_candidate - ON fts_candidate.project_id = search_index.project_id - AND fts_candidate.id = search_index.id - AND fts_candidate.type = search_index.type - """ - conditions.extend( - "(search_index.script_ngrams_index_col " - f"@@ to_tsquery('simple', :script_text_{index}) OR EXISTS (" - "SELECT 1 FROM search_index_fts_chunks AS script_chunk " - "WHERE script_chunk.project_id = search_index.project_id " - "AND script_chunk.search_index_id = search_index.id " - "AND script_chunk.search_index_type = search_index.type " - "AND script_chunk.script_ngrams_index_col " - f"@@ to_tsquery('simple', :script_text_{index})))" - for index in range(len(script_tsqueries)) - ) - - # Handle title search - if title: - title_text = self._prepare_search_term(title.strip(), is_prefix=False) - params["title_text"] = title_text - conditions.append( - "to_tsvector('english', search_index.title) @@ to_tsquery('english', :title_text)" - ) - - # Handle permalink exact search - if permalink: - params["permalink"] = permalink - conditions.append("search_index.permalink = :permalink") - - # Handle permalink pattern match - if permalink_match: - permalink_text = permalink_match.lower().strip() - params["permalink"] = permalink_text - if "*" in permalink_match: - # Use LIKE for pattern matching in Postgres - # Convert * to % for SQL LIKE - permalink_pattern = permalink_text.replace("*", "%") - params["permalink"] = permalink_pattern - conditions.append("search_index.permalink LIKE :permalink") - else: - conditions.append("search_index.permalink = :permalink") - - # Handle directory subtree scope. The predicate is built by the shared - # helper so Postgres and SQLite scope by the identical rule; see - # file_path_prefix_condition for the boundary and escaping reasoning. - subtree_condition = file_path_prefix_condition(file_path_prefix, params) - if subtree_condition is not None: - conditions.append(subtree_condition) - - # Handle an explicit candidate-row restriction. Built by the shared helper so - # both backends restrict by the identical rule; see - # candidate_key_restriction_condition for why the vector filter pass asks about - # its candidates rather than paging the filter's whole match set (#1431). - if candidate_keys is not None: - conditions.append(candidate_key_restriction_condition(candidate_keys, params)) - - # Handle search item type filter (parameterized for defense-in-depth) - if search_item_types: - type_placeholders = [] - for idx, t in enumerate(search_item_types): - param_name = f"search_type_{idx}" - params[param_name] = t.value - type_placeholders.append(f":{param_name}") - conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})") - - # Handle observation category filter (parameterized for defense-in-depth). - # Trigger: caller passed `categories` to scope observation results. - # Why: `entity_types=["observation"]` only narrows to the observation row type; - # callers expect exact-category matching, not incidental text matches. - # Outcome: only rows whose indexed category exactly equals a requested value - # survive (entities/relations have NULL category and are excluded). - if categories: - category_placeholders = [] - for idx, category in enumerate(categories): - param_name = f"category_{idx}" - params[param_name] = category - category_placeholders.append(f":{param_name}") - conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})") - - # Handle note type filter (frontmatter type field, parameterized). - # Trigger: caller passed `note_types` to scope by the frontmatter `type` field. - # Why: the type belongs to the note, but only its entity row carries the - # frontmatter; observation and relation rows do not. Reading it off each row - # silently excluded every non-entity row, which made `note_types` combined - # with a valid-time filter unsatisfiable. - # Outcome: resolved through the owning note in one shared builder, so both - # backends ask the same question and observation rows of a matching note - # are admitted. - if note_types: - conditions.append( - build_note_type_predicate( - note_types, params, note_type_value=POSTGRES_NOTE_TYPE_VALUE - ) - ) - - # Handle date filter - if after_date: - params["after_date"] = after_date - # Filter on updated_at so recently-edited notes are included even when created_at is old - conditions.append("search_index.updated_at > :after_date") - # order by most recent first - order_by_clause = ", search_index.updated_at DESC" - - # Handle authored valid time (SPEC-82). - # Trigger: caller asked when a statement was true of the world. - # Why: `after_date` above filters `updated_at`, which records when the note was - # last edited. That is bookkeeping, never a semantic claim; a decision - # effective through July says nothing about when its file was touched. - # Outcome: an independent predicate over the temporal projection, textually - # identical to the SQLite one because canonical bounds compare - # lexicographically on both backends. Undated sources carry no row and - # are therefore excluded whenever a valid-time filter is present. - if temporal is not None: - conditions.append(build_temporal_predicate(temporal, params)) - - # Handle structured metadata filters (frontmatter) - # Uses jsonb_extract_path_text() / jsonb_extract_path() with parameterized - # path parts instead of #>> / #> with interpolated paths. - if metadata_filters: - parsed_filters = parse_metadata_filters(metadata_filters) - from_clause = f"{from_clause} JOIN entity ON search_index.entity_id = entity.id" - # Frontmatter filters answer for notes only; see - # metadata_filter_content_type_condition for why every regular file - # would otherwise satisfy a null predicate. - conditions.append(metadata_filter_content_type_condition(params)) - metadata_expr = "entity.entity_metadata::jsonb" - - for idx, filt in enumerate(parsed_filters): - # Parameterize each JSON path part individually - path_param_names = [] - for j, part in enumerate(filt.path_parts): - path_param = f"meta_path_{idx}_{j}" - params[path_param] = part - path_param_names.append(f":{path_param}") - path_args = ", ".join(path_param_names) - text_expr = f"jsonb_extract_path_text({metadata_expr}, {path_args})" - json_expr = f"jsonb_extract_path({metadata_expr}, {path_args})" - - # jsonb_extract_path_text returns SQL NULL both for a missing key - # and for an explicit JSON null — the same two cases SQLite's - # json_extract collapses — so the dialects answer - # `{"owner": None}` row for row. `= NULL` is never true, so - # equality here would report a confident zero. - if filt.op == "is_null": - conditions.append(f"{text_expr} IS NULL") - continue - - if filt.op == "eq": - value_param = f"meta_val_{idx}" - params[value_param] = filt.value - conditions.append(f"{text_expr} = :{value_param}") - continue - - if filt.op == "in": - placeholders = [] - for j, val in enumerate(filt.value): - value_param = f"meta_val_{idx}_{j}" - params[value_param] = val - placeholders.append(f":{value_param}") - conditions.append(f"{text_expr} IN ({', '.join(placeholders)})") - continue - - if filt.op == "contains": - base_param = f"meta_val_{idx}" - tag_conditions = [] - # Require all values to be present - for j, val in enumerate(filt.value): - tag_param = f"{base_param}_{j}" - params[tag_param] = json.dumps([val]) - # The exact JSONB containment test is the primary path; the - # substring patterns only reach values stored as array text. - like_condition = metadata_contains_like_condition( - text_expr, - val, - param_prefix=tag_param, - params=params, - ) - tag_conditions.append( - f"({json_expr} @> CAST(:{tag_param} AS jsonb) OR {like_condition})" - ) - conditions.append(" AND ".join(tag_conditions)) - continue - - if filt.op in {"gt", "gte", "lt", "lte", "between"}: - compare_expr = ( - f"{text_expr}::double precision" - if filt.comparison == "numeric" - else text_expr - ) - - if filt.op == "between": - min_param = f"meta_val_{idx}_min" - max_param = f"meta_val_{idx}_max" - params[min_param] = filt.value[0] - params[max_param] = filt.value[1] - conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}") - else: - value_param = f"meta_val_{idx}" - params[value_param] = filt.value - operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op] - conditions.append(f"{compare_expr} {operator} :{value_param}") - continue - - # Always filter by project_id - params["project_id"] = self.project_id - conditions.append("search_index.project_id = :project_id") - - # Build WHERE clause - where_clause = " AND ".join(conditions) if conditions else "1=1" - - # Build SQL with ts_rank() for scoring - # Note: If no text search, score will be NULL, so we use COALESCE to default to 0 - score_parts: list[str] = [] - if document_vector_sql is not None: - score_parts.append( - "GREATEST(" - f"ts_rank({document_vector_sql}, to_tsquery('english', :text)), " - "ts_rank(search_index.textsearchable_index_col, to_tsquery('english', :text)), " - "COALESCE((SELECT MAX(ts_rank(" - "fts_chunk.textsearchable_index_col, to_tsquery('english', :text))) " - "FROM search_index_fts_chunks AS fts_chunk " - "WHERE fts_chunk.project_id = search_index.project_id " - "AND fts_chunk.search_index_id = search_index.id " - "AND fts_chunk.search_index_type = search_index.type " - "AND fts_chunk.textsearchable_index_col " - "@@ to_tsquery('english', :text)), 0))" - ) - score_parts.extend( - "GREATEST(" - "ts_rank(search_index.script_ngrams_index_col, " - f"to_tsquery('simple', :script_text_{index})), " - "COALESCE((SELECT MAX(ts_rank(script_rank.script_ngrams_index_col, " - f"to_tsquery('simple', :script_text_{index}))) " - "FROM search_index_fts_chunks AS script_rank " - "WHERE script_rank.project_id = search_index.project_id " - "AND script_rank.search_index_id = search_index.id " - "AND script_rank.search_index_type = search_index.type " - "AND script_rank.script_ngrams_index_col " - f"@@ to_tsquery('simple', :script_text_{index})), 0))" - for index in range(len(script_tsqueries)) - ) - # Each condition above is required, so every query component should contribute to - # relevance. Taking only the strongest rank makes additional script runs invisible. - score_expr = " + ".join(score_parts) if score_parts else "0" - - return from_clause, where_clause, params, order_by_clause, score_expr - - @staticmethod - def _document_fts_vector_sql(processed_texts: Sequence[str], params: dict[str, Any]) -> str: - """Build a query-sized vector representing lexemes found anywhere in one item.""" - operands: dict[str, str] = {} - for processed_text in processed_texts: - for operand, representative in _tsquery_operands(processed_text): - operands.setdefault(operand, representative) - - present_lexemes: list[str] = [] - for index, (operand, representative) in enumerate(operands.items()): - operand_param = f"text_operand_{index}" - representative_param = f"text_representative_{index}" - params[operand_param] = operand - params[representative_param] = representative - present_lexemes.append( - "CASE WHEN (search_index.textsearchable_index_col " - f"@@ to_tsquery('english', :{operand_param}) OR EXISTS (" - "SELECT 1 FROM search_index_fts_chunks AS operand_chunk " - "WHERE operand_chunk.project_id = search_index.project_id " - "AND operand_chunk.search_index_id = search_index.id " - "AND operand_chunk.search_index_type = search_index.type " - "AND operand_chunk.textsearchable_index_col " - f"@@ to_tsquery('english', :{operand_param}))) " - f"THEN :{representative_param} ELSE '' END" - ) - - if not present_lexemes: - return "search_index.textsearchable_index_col" - - # The synthesized text contains at most the query operands, never the note body. - # This preserves document-wide Boolean semantics without recreating an unbounded vector. - lexeme_array = f"ARRAY[{', '.join(present_lexemes)}]" - return f"to_tsvector('english', array_to_string({lexeme_array}, ' '))" - @override async def search( self, @@ -1430,13 +664,8 @@ async def search( return dispatched # --- FTS mode (Postgres-specific) --- - ( - from_clause, - where_clause, - params, - order_by_clause, - score_expr, - ) = await self._build_fts_query_parts( + compiled = compile_fts_filter( + self.scope, search_text=search_text, permalink=permalink, permalink_match=permalink_match, @@ -1451,8 +680,7 @@ async def search( allow_relaxed=allow_relaxed, candidate_keys=candidate_keys, ) - - # set limit and offset + params = compiled.params params["limit"] = limit params["offset"] = offset @@ -1473,10 +701,10 @@ async def search( search_index.category, search_index.created_at, search_index.updated_at, - {score_expr} as score - FROM {from_clause} - WHERE {where_clause} - ORDER BY score DESC {order_by_clause}, search_index.id ASC + {compiled.score_expression} as score + FROM {compiled.from_clause} + WHERE {compiled.where_clause} + ORDER BY score DESC {compiled.order_by_clause}, search_index.id ASC LIMIT :limit OFFSET :offset """ @@ -1498,13 +726,13 @@ async def execute_rows(active_session: AsyncSession, query_params: dict[str, Any return result.fetchall() async def run_search(active_session: AsyncSession): - relaxed = self._relaxed_tsquery_text(search_text) if allow_relaxed else None + relaxed = relaxed_tsquery_text(search_text) if allow_relaxed else None strict_syntax_error = False relaxed_fallback_used = False try: rows = await execute_rows(active_session, params) except Exception as exc: - if not (self._is_tsquery_syntax_error(exc) and relaxed and params.get("text")): + if not (is_tsquery_syntax_error(exc) and relaxed and params.get("text")): raise strict_syntax_error = True rows = [] @@ -1545,7 +773,7 @@ async def run_search(active_session: AsyncSession): async with db.scoped_session(self.session_maker) as owned_session: rows, relaxed_fallback_used = await run_search(owned_session) except Exception as e: - if self._is_tsquery_syntax_error(e): + if is_tsquery_syntax_error(e): logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}") if trace is not None: trace.fts = build_fts_page_stage( @@ -1619,13 +847,8 @@ async def count( min_similarity=min_similarity, ) - ( - from_clause, - where_clause, - params, - _order_by_clause, - _score_expr, - ) = await self._build_fts_query_parts( + compiled = compile_fts_filter( + self.scope, search_text=search_text, permalink=permalink, permalink_match=permalink_match, @@ -1639,7 +862,8 @@ async def count( temporal=temporal, allow_relaxed=allow_relaxed, ) - sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" + params = compiled.params + sql = f"SELECT COUNT(*) FROM {compiled.from_clause} WHERE {compiled.where_clause}" logger.trace(f"Count {sql} params: {params}") async def execute_count(active_session: AsyncSession, query_params: dict[str, Any]) -> int: @@ -1652,12 +876,12 @@ async def execute_count(active_session: AsyncSession, query_params: dict[str, An try: async with db.scoped_session(self.session_maker) as session: - relaxed = self._relaxed_tsquery_text(search_text) if allow_relaxed else None + relaxed = relaxed_tsquery_text(search_text) if allow_relaxed else None strict_syntax_error = False try: total = await execute_count(session, params) except Exception as exc: - if not (self._is_tsquery_syntax_error(exc) and relaxed and params.get("text")): + if not (is_tsquery_syntax_error(exc) and relaxed and params.get("text")): raise strict_syntax_error = True total = 0 @@ -1675,7 +899,7 @@ async def execute_count(active_session: AsyncSession, query_params: dict[str, An ) return total except Exception as e: - if self._is_tsquery_syntax_error(e): + if is_tsquery_syntax_error(e): logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}") return 0 logger.error(f"Database error during search count: {e}") diff --git a/src/basic_memory/repository/search_filters.py b/src/basic_memory/repository/search_filters.py new file mode 100644 index 000000000..0a3233bca --- /dev/null +++ b/src/basic_memory/repository/search_filters.py @@ -0,0 +1,147 @@ +"""WHERE-clause pieces both search backends share. + +The two FTS engines differ in how they match text and read JSON. Every other filter a +search accepts asks the same question of the same columns on both, so it is compiled +once here. A backend supplies the two spellings that differ through ``FilterDialect`` +and appends its own text, title, permalink-pattern, and metadata predicates around the +shared ones. +""" + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +from basic_memory.repository.note_type_filters import ( + POSTGRES_NOTE_TYPE_VALUE, + SQLITE_NOTE_TYPE_VALUE, + build_note_type_predicate, +) +from basic_memory.repository.search_repository_base import ( + SearchIndexKey, + candidate_key_restriction_condition, + file_path_prefix_condition, +) +from basic_memory.repository.search_scope import ProjectScope +from basic_memory.repository.temporal_filters import build_temporal_predicate +from basic_memory.schemas.search import SearchItemType +from basic_memory.temporal import TemporalFilter + +# Newest edits first whenever the caller filtered on ``after_date``. +AFTER_DATE_ORDER_BY = ", search_index.updated_at DESC" + + +@dataclass(frozen=True, slots=True) +class FilterDialect: + """The two SQL spellings that differ between backends inside the shared filters.""" + + note_type_value: str + after_date_condition: str + + +SQLITE_FILTER_DIALECT = FilterDialect( + note_type_value=SQLITE_NOTE_TYPE_VALUE, + # datetime() normalizes both sides so ISO strings of mixed precision compare as instants. + after_date_condition="datetime(search_index.updated_at) > datetime(:after_date)", +) +POSTGRES_FILTER_DIALECT = FilterDialect( + note_type_value=POSTGRES_NOTE_TYPE_VALUE, + after_date_condition="search_index.updated_at > :after_date", +) + + +@dataclass(frozen=True, slots=True) +class CompiledFilter: + """One backend's FROM, WHERE, and score for a search, ready to place in a statement. + + ``params`` is the bind dictionary the statement runs with. Callers add ``limit`` + and ``offset``, and a relaxed retry replaces ``text``. + """ + + from_clause: str + where_clause: str + params: dict[str, Any] + order_by_clause: str + score_expression: str + + +def shared_filter_conditions( + scope: ProjectScope, + params: dict[str, Any], + *, + dialect: FilterDialect, + permalink: str | None, + file_path_prefix: str | None, + candidate_keys: Sequence[SearchIndexKey] | None, + search_item_types: Sequence[SearchItemType] | None, + categories: Sequence[str] | None, + note_types: Sequence[str] | None, + after_date: datetime | None, + temporal: TemporalFilter | None, +) -> list[str]: + """Compile the filters whose SQL is identical on both backends. + + Binds are added to ``params`` in place. The scope predicate comes first: it is the + one filter every statement carries, and it is what keeps rows outside the caller's + projects out of every candidate window. + """ + conditions = [scope.predicate("search_index.project_id", params)] + + if permalink: + params["permalink"] = permalink + conditions.append("search_index.permalink = :permalink") + + # See file_path_prefix_condition for the subtree boundary and escaping rules. + subtree_condition = file_path_prefix_condition(file_path_prefix, params) + if subtree_condition is not None: + conditions.append(subtree_condition) + + # See candidate_key_restriction_condition for why the vector filter pass asks + # about its candidates rather than paging the filter's whole match set (#1431). + if candidate_keys is not None: + conditions.append(candidate_key_restriction_condition(candidate_keys, params)) + + if search_item_types: + type_placeholders: list[str] = [] + for index, item_type in enumerate(search_item_types): + name = f"search_type_{index}" + params[name] = item_type.value + type_placeholders.append(f":{name}") + conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})") + + # Trigger: caller passed ``categories`` to scope observation results. + # Why: ``entity_types=["observation"]`` only narrows to the observation row type; + # callers expect exact-category matching, not incidental text matches. + # Outcome: only rows whose indexed category exactly equals a requested value + # survive (entities/relations have NULL category and are excluded). + if categories: + category_placeholders: list[str] = [] + for index, category in enumerate(categories): + name = f"category_{index}" + params[name] = category + category_placeholders.append(f":{name}") + conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})") + + # The note type belongs to the note, but only its entity row carries the + # frontmatter, so the predicate resolves through the owning note. See + # note_type_filters for why reading it off each row excluded every non-entity row. + if note_types: + conditions.append( + build_note_type_predicate( + note_types, params, scope=scope, note_type_value=dialect.note_type_value + ) + ) + + # Filter on updated_at so recently edited notes are included even when created_at + # is old. The matching ORDER BY lives in AFTER_DATE_ORDER_BY. + if after_date: + params["after_date"] = after_date + conditions.append(dialect.after_date_condition) + + # Authored valid time (SPEC-82) is independent of ``after_date``: that one is + # bookkeeping about the file, this one is a claim about the world. See + # temporal_filters for the overlap rule and why the subquery is non-correlated. + if temporal is not None: + conditions.append(build_temporal_predicate(temporal, params, scope=scope)) + + return conditions diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 7e824a86e..99f8b23ae 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -33,6 +33,7 @@ validate_rerank_scores, ) from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.search_scope import ProjectScope from basic_memory.repository.script_ngrams import build_script_ngrams from basic_memory.repository.search_trace import ( BelowThreshold, @@ -405,6 +406,8 @@ def __init__(self, session_maker: async_sessionmaker[AsyncSession], project_id: self.session_maker = session_maker self.project_id = project_id + # Every statement this repository compiles reads exactly one project. + self.scope = ProjectScope.single(project_id) async def semantic_effectively_enabled(self) -> bool: """Return whether semantic retrieval can actually run for this repository. @@ -455,23 +458,6 @@ async def init_search_index(self) -> None: """ pass - @abstractmethod - def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: - """Prepare a search term for backend-specific query syntax. - - Args: - term: The search term to prepare - is_prefix: Whether to add prefix search capability - - Returns: - Formatted search term for the backend - - Backend-specific implementations: - - SQLite: Quotes FTS5 special characters, adds * wildcards - - Postgres: Converts to tsquery syntax with :* prefix operator - """ - pass - @abstractmethod async def search( self, diff --git a/src/basic_memory/repository/search_scope.py b/src/basic_memory/repository/search_scope.py new file mode 100644 index 000000000..c6fa78137 --- /dev/null +++ b/src/basic_memory/repository/search_scope.py @@ -0,0 +1,61 @@ +"""The project set a search statement is allowed to read. + +Every search row, vector chunk, and temporal assertion carries a ``project_id``. A +statement binds that column to an explicit set before any ranking runs, so rows +outside the set never occupy a candidate window. Absence is not a value here: a scope +is always built from concrete IDs, and an empty scope compiles to a predicate that +admits nothing. +""" + +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +# No project can match, and no bind needs to be sent to prove it. +_MATCHES_NOTHING = "1 = 0" + + +@dataclass(frozen=True, slots=True) +class ProjectScope: + """Unique, positive project IDs a statement may read, in ascending order. + + Build one with ``ProjectScope.of`` or ``ProjectScope.single``; both canonicalize + the input so two scopes over the same projects compare equal. + """ + + project_ids: tuple[int, ...] + + def __post_init__(self) -> None: + for project_id in self.project_ids: + # bool is an int subclass; ``True`` would silently read as project 1. + if isinstance(project_id, bool) or not isinstance(project_id, int) or project_id <= 0: + raise ValueError(f"Project IDs must be positive integers, got {project_id!r}") + + @classmethod + def of(cls, project_ids: Iterable[int]) -> "ProjectScope": + """Canonicalize any iterable of project IDs into a scope.""" + return cls(tuple(sorted(set(project_ids)))) + + @classmethod + def single(cls, project_id: int) -> "ProjectScope": + """The scope every project-bound repository runs under.""" + return cls((project_id,)) + + @property + def is_empty(self) -> bool: + return not self.project_ids + + def predicate(self, column: str, params: dict[str, Any]) -> str: + """SQL restricting ``column`` to this scope, adding its binds to ``params``. + + The bind names are a function of the scope alone, so a statement that + references the scope from several subqueries sends each ID once. + """ + if self.is_empty: + return _MATCHES_NOTHING + placeholders: list[str] = [] + for index, project_id in enumerate(self.project_ids): + name = f"scope_{index}" + params[name] = project_id + placeholders.append(f":{name}") + return f"{column} IN ({', '.join(placeholders)})" diff --git a/src/basic_memory/repository/sqlite_search_query.py b/src/basic_memory/repository/sqlite_search_query.py new file mode 100644 index 000000000..9fa161bab --- /dev/null +++ b/src/basic_memory/repository/sqlite_search_query.py @@ -0,0 +1,423 @@ +"""SQLite FTS5 query preparation: term syntax and filter compilation. + +Pure functions over a ``ProjectScope`` and the caller's filters. Nothing here opens +a session or owns an index; the repository resolves the one piece of live schema +state the compiler needs (the entity table's columns) and passes it in. +""" + +import re +from collections.abc import Collection, Sequence +from datetime import datetime +from typing import Any + +from basic_memory.repository.metadata_filters import build_sqlite_json_path, parse_metadata_filters +from basic_memory.repository.script_ngrams import analyze_script_query +from basic_memory.repository.search_filters import ( + AFTER_DATE_ORDER_BY, + SQLITE_FILTER_DIALECT, + CompiledFilter, + shared_filter_conditions, +) +from basic_memory.repository.search_query import relaxed_query_words +from basic_memory.repository.search_repository_base import ( + SearchIndexKey, + metadata_contains_like_condition, + metadata_filter_content_type_condition, +) +from basic_memory.repository.search_scope import ProjectScope +from basic_memory.schemas.search import SearchItemType +from basic_memory.temporal import TemporalFilter + +SQLITE_WORD_COLUMNS = "{title content_stems content_snippet}" + +# Characters that indicate a term should be quoted (parentheses excluded: valid syntax). +_NEEDS_QUOTING_CHARS = frozenset(" .:;,<>?/-'\"[]{}+!@#$%^&=|\\~`") +# Characters that can cause FTS5 syntax errors when read as operators. +_PROBLEMATIC_CHARS = frozenset("\"'()[]{}+!@#$%^&=|\\~`") +# Characters that indicate quoting for spaces, dots, colons, and hyphens followed by +# wildcards, which FTS5 mishandles. +_SPACE_OR_SPECIAL_CHARS = frozenset(" .:;,<>?/-") +_BOOLEAN_OPERATOR_PATTERN = r"(\bAND\b|\bOR\b|\bNOT\b)" + + +# --- Term preparation --- + + +def needs_quoting(term: str) -> bool: + """Whether a term must be quoted for FTS5 safety.""" + if not term or not term.strip(): + return False + return any(c in _NEEDS_QUOTING_CHARS for c in term) + + +def prepare_single_term(term: str, is_prefix: bool = True) -> str: + """Prepare one search term with no Boolean operators. + + ``is_prefix`` adds the ``*`` suffix so simple terms match by prefix. + """ + if not term or not term.strip(): + return term + + term = term.strip() + + # A proper wildcard pattern ("hello*", "test*world") is left alone. + if "*" in term and all(c.isalnum() or c in "*_-" for c in term): + return term + + # Natural-language queries arrive with sentence punctuation that FTS5 treats as + # syntax ("When did Melanie paint a sunrise?"). The tokenizer ignores this + # punctuation in the index, so stripping it from word edges loses nothing, but + # leaving it forces the whole question into an exact-phrase match that returns + # zero rows and silently disables the FTS half of hybrid search. Interior + # characters (hyphens, slashes in permalinks and paths) are untouched. + if " " in term: + words = [word.strip("?!.,;:") for word in term.split()] + term = " ".join(word for word in words if word) + if not term: + return "" + + has_problematic = any(c in _PROBLEMATIC_CHARS for c in term) + has_spaces_or_special = any(c in _SPACE_OR_SPECIAL_CHARS for c in term) + + if has_problematic or has_spaces_or_special: + if " " in term and not has_problematic: + words = term.split() + has_special_in_words = any( + any(c in word for c in _SPACE_OR_SPECIAL_CHARS if c != " ") for word in words + ) + if not has_special_in_words: + # Multi-word queries of simple words ("emoji unicode") use Boolean AND + # so word order does not matter. + prepared_words = [f"{word}*" for word in words] if is_prefix else words + return " AND ".join(prepared_words) + # Any word with special characters quotes the entire phrase. + escaped_term = term.replace('"', '""') + if is_prefix and not ("/" in term and term.endswith(".md")): + return f'"{escaped_term}"*' + return f'"{escaped_term}"' # pragma: no cover + + # Terms with problematic characters or file paths use exact phrase matching. + escaped_term = term.replace('"', '""') + if is_prefix and not ("/" in term and term.endswith(".md")): + return f'"{escaped_term}"*' + return f'"{escaped_term}"' + + if is_prefix: + return f"{term}*" + return term + + +def prepare_parenthetical_term(term: str) -> str: + """Prepare a term containing parentheses, preserving them for grouping.""" + result = "" + index = 0 + while index < len(term): + if term[index] in "()": + result += term[index] + index += 1 + continue + start = index + while index < len(term) and term[index] not in "()": + index += 1 + content = term[start:index].strip() + if content: + # Quote only when the content needs it; simple words stay bare. + if needs_quoting(content): + escaped_content = content.replace('"', '""') + result += f'"{escaped_content}"' + else: + result += content + return result + + +def prepare_boolean_query(query: str) -> str: + """Quote the terms of a Boolean query while preserving its operators and grouping.""" + processed_parts: list[str] = [] + for part in re.split(_BOOLEAN_OPERATOR_PATTERN, query): + part = part.strip() + if not part: + continue + if part in ("AND", "OR", "NOT"): + processed_parts.append(part) + elif "(" in part or ")" in part: + processed_parts.append(prepare_parenthetical_term(part)) + else: + # Boolean queries do not get prefix wildcards. + processed_parts.append(prepare_single_term(part, is_prefix=False)) + return " ".join(processed_parts) + + +def prepare_search_term(term: str, is_prefix: bool = True) -> str: + """Prepare user text as an FTS5 query. + + Boolean operators (AND, OR, NOT) are preserved. Terms with FTS5 special + characters are quoted. Simple terms get prefix wildcards. + """ + if any(op in f" {term} " for op in (" AND ", " OR ", " NOT ")): + return prepare_boolean_query(term) + return prepare_single_term(term, is_prefix) + + +def _relaxed_fts_term(word: str) -> str: + """Render one relaxed word as an FTS5-safe prefix expression. + + A word token can contain an apostrophe ("об'єкт", "don't"). Interpolated bare it + is FTS5 syntax, not text: the whole expression fails to parse, the caller swallows + the syntax error, and the relaxed retry returns nothing, which is the silent-empty + FTS failure this fallback exists to prevent. + """ + if "'" in word or '"' in word: + return '"{}"*'.format(word.replace('"', '""')) + return f"{word}*" + + +def relaxed_fts_text(search_text: str | None) -> str | None: + """OR-relaxed FTS5 expression for a failed strict query, or None.""" + words = relaxed_query_words(search_text) + if not words: + return None + return " OR ".join(_relaxed_fts_term(word) for word in words) + + +def is_fts5_syntax_error(exc: Exception) -> bool: + return "fts5: syntax error" in str(exc).lower() + + +# --- Filter compilation --- + + +def compile_fts_filter( + scope: ProjectScope, + *, + entity_columns: Collection[str], + search_text: str | None = None, + permalink: str | None = None, + permalink_match: str | None = None, + title: str | None = None, + note_types: Sequence[str] | None = None, + after_date: datetime | None = None, + search_item_types: Sequence[SearchItemType] | None = None, + categories: Sequence[str] | None = None, + metadata_filters: dict[str, Any] | None = None, + file_path_prefix: str | None = None, + temporal: TemporalFilter | None = None, + candidate_keys: Sequence[SearchIndexKey] | None = None, +) -> CompiledFilter: + """Compile SQLite FTS FROM/WHERE/score shared by search and count. + + ``entity_columns`` is the live column set of the ``entity`` table. Generated + frontmatter columns are used when present and fall back to ``json_extract``. + """ + params: dict[str, Any] = {} + conditions = shared_filter_conditions( + scope, + params, + dialect=SQLITE_FILTER_DIALECT, + permalink=permalink, + file_path_prefix=file_path_prefix, + candidate_keys=candidate_keys, + search_item_types=search_item_types, + categories=categories, + note_types=note_types, + after_date=after_date, + temporal=temporal, + ) + match_conditions: list[str] = [] + from_clause = "search_index" + score_expression = "bm25(search_index)" + preserve_match_score = False + + # Wildcard-only and blank text add no text condition: every row matches. + if search_text and search_text.strip() not in ("", "*"): + script_query = analyze_script_query(search_text.strip()) + # Trigger: the query contains text from an unsegmented script. + # Why: the script channel needs one table-level MATCH alongside word fields. + # Outcome: mixed queries rank all terms together; word-only queries retain + # their established per-column matching and ranking behavior. + if script_query.gram_phrases: + preserve_match_score = True + params["text"] = "" + params["script_text"] = "" + if script_query.word_text: + prepared_text = prepare_search_term(script_query.word_text) + params["text"] = ( + f"(title: ({prepared_text}) OR " + f"content_stems: ({prepared_text}) OR " + f"content_snippet: ({prepared_text}))" + ) + script_phrases = " AND ".join( + f'"{" ".join(phrase)}"' for phrase in script_query.gram_phrases + ) + script_clause = f"script_ngrams: ({script_phrases})" + params["script_text"] = ( + f" AND ({script_clause})" if script_query.word_text else script_clause + ) + match_conditions.append("search_index MATCH (:text || :script_text)") + else: + word_text = ( + script_query.word_text + if script_query.word_text is not None + else search_text.strip() + ) + params["text"] = prepare_search_term(word_text) + # content_stems is capped for Postgres index-row compatibility, while + # SQLite stores the complete note body in its FTS5 content_snippet column. + match_conditions.append( + "(search_index.title MATCH :text OR " + "search_index.content_stems MATCH :text OR " + "search_index.content_snippet MATCH :text)" + ) + + if title: + params["title_text"] = prepare_search_term(title.strip(), is_prefix=False) + match_conditions.append("search_index.title MATCH :title_text") + + if permalink_match: + # GLOB patterns keep their syntax; prepare_search_term would quote the slashes. + permalink_text = permalink_match.lower().strip() + params["permalink"] = permalink_text + if "*" in permalink_match: + conditions.append("search_index.permalink GLOB :permalink") + elif "/" in permalink_text: + conditions.append("search_index.permalink = :permalink") + else: + # A bare name without a path matches through FTS5. + params["permalink"] = prepare_search_term(permalink_text, is_prefix=False) + match_conditions.append("search_index.permalink MATCH :permalink") + + if metadata_filters: + parsed_filters = parse_metadata_filters(metadata_filters) + from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id" + # Frontmatter filters answer for notes only; see + # metadata_filter_content_type_condition for why every regular file would + # otherwise satisfy a null predicate. + conditions.append(metadata_filter_content_type_condition(params)) + + for idx, filt in enumerate(parsed_filters): + path_param = f"meta_path_{idx}" + extract_expr = None + use_tags_column = False + + if filt.path_parts == ["status"] and "frontmatter_status" in entity_columns: + extract_expr = "entity.frontmatter_status" + elif filt.path_parts == ["type"] and "frontmatter_type" in entity_columns: + extract_expr = "entity.frontmatter_type" + elif filt.path_parts == ["tags"] and "tags_json" in entity_columns: + extract_expr = "entity.tags_json" + use_tags_column = True + + if extract_expr is None: + params[path_param] = build_sqlite_json_path(filt.path_parts) + extract_expr = f"json_extract(entity.entity_metadata, :{path_param})" + + # json_extract returns SQL NULL both for a missing key and for an explicit + # JSON null, and the generated frontmatter_* columns are that same + # json_extract, so IS NULL means "the note carries no value here", the + # question ``{"owner": None}`` asks. ``= NULL`` is never true. + if filt.op == "is_null": + conditions.append(f"{extract_expr} IS NULL") + continue + + if filt.op == "eq": + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + conditions.append(f"{extract_expr} = :{value_param}") + continue + + if filt.op == "in": + placeholders: list[str] = [] + for j, val in enumerate(filt.value): + value_param = f"meta_val_{idx}_{j}" + params[value_param] = val + placeholders.append(f":{value_param}") + conditions.append(f"{extract_expr} IN ({', '.join(placeholders)})") + continue + + if filt.op == "contains": + tag_conditions: list[str] = [] + for j, val in enumerate(filt.value): + value_param = f"meta_val_{idx}_{j}" + params[value_param] = val + # The exact JSON-membership test is the primary path; the + # substring patterns only reach values stored as array text. + like_condition = metadata_contains_like_condition( + extract_expr, + val, + param_prefix=value_param, + params=params, + ) + json_each_expr = ( + "json_each(entity.tags_json)" + if use_tags_column + else f"json_each(entity.entity_metadata, :{path_param})" + ) + tag_conditions.append( + "(" + f"EXISTS (SELECT 1 FROM {json_each_expr} WHERE value = :{value_param}) " + f"OR {like_condition}" + ")" + ) + conditions.append(" AND ".join(tag_conditions)) + continue + + if filt.op in {"gt", "gte", "lt", "lte", "between"}: + compare_expr = ( + f"CAST({extract_expr} AS REAL)" + if filt.comparison == "numeric" + else extract_expr + ) + if filt.op == "between": + min_param = f"meta_val_{idx}_min" + max_param = f"meta_val_{idx}_max" + params[min_param] = filt.value[0] + params[max_param] = filt.value[1] + conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}") + else: + value_param = f"meta_val_{idx}" + params[value_param] = filt.value + operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op] + conditions.append(f"{compare_expr} {operator} :{value_param}") + continue + + # Trigger: SQLite rejects some Boolean combinations of MATCH predicates, + # including a word-field OR expression combined with the script channel. + # Why: each MATCH must be evaluated in an FTS-valid query context. + # Outcome: keep one outer MATCH for bm25 ranking and intersect the rest by rowid. + if len(match_conditions) > 1: + ranked_match, *additional_matches = match_conditions + conditions.extend( + f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_condition})" + for match_condition in additional_matches + ) + match_conditions = [ranked_match] + + # Trigger: SQLite FTS MATCH predicates combined with JOINs can fail with + # "unable to use function MATCH in the requested context". + # Why: script queries need MATCH and bm25 together for ranking, while legacy + # word-column OR predicates cannot evaluate bm25 in the same derived query. + # Outcome: rank script matches before joining metadata; retain the established + # rowid-filter path for word-only searches. + if metadata_filters and match_conditions: + match_where = " AND ".join(match_conditions) + if preserve_match_score: + from_clause = ( + "(SELECT search_index.rowid AS rowid, search_index.*, " + "bm25(search_index) AS fts_score " + f"FROM search_index WHERE {match_where}) AS search_index " + "JOIN entity ON search_index.entity_id = entity.id" + ) + score_expression = "search_index.fts_score" + else: + conditions.append( + f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_where})" + ) + else: + conditions.extend(match_conditions) + + return CompiledFilter( + from_clause=from_clause, + where_clause=" AND ".join(conditions), + params=params, + order_by_clause=AFTER_DATE_ORDER_BY if after_date else "", + score_expression=score_expression, + ) diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 4160d7932..11bb39ff0 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -1,7 +1,6 @@ """SQLite FTS5-based search repository implementation.""" import asyncio -import re import time from collections.abc import Sequence from contextlib import asynccontextmanager @@ -31,22 +30,18 @@ from basic_memory.repository.search_repository_base import ( SearchIndexKey, SearchRepositoryBase, - candidate_key_restriction_condition, - file_path_prefix_condition, - metadata_contains_like_condition, - metadata_filter_content_type_condition, ) from basic_memory.repository.script_ngrams import analyze_script_query from basic_memory.repository.search_trace import ( SearchTraceCollector, build_fts_page_stage, ) -from basic_memory.repository.metadata_filters import parse_metadata_filters, build_sqlite_json_path -from basic_memory.repository.note_type_filters import ( - SQLITE_NOTE_TYPE_VALUE, - build_note_type_predicate, +from basic_memory.repository.sqlite_search_query import ( + SQLITE_WORD_COLUMNS, + compile_fts_filter, + is_fts5_syntax_error, + relaxed_fts_text, ) -from basic_memory.repository.temporal_filters import build_temporal_predicate from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError from basic_memory.repository.semantic_vector_index import SemanticVectorIndex from basic_memory.repository.semantic_vector_sync import StagedVectorDeletion @@ -56,9 +51,6 @@ from basic_memory.temporal import TemporalFilter -SQLITE_WORD_COLUMNS = "{title content_stems content_snippet}" - - class SQLiteSearchRepository(SearchRepositoryBase): """SQLite FTS5 implementation of search repository. @@ -160,281 +152,6 @@ async def init_search_index(self): ) self._semantic_enabled = False - # ------------------------------------------------------------------ - # FTS5 query preparation (backend-specific) - # ------------------------------------------------------------------ - - def _prepare_boolean_query(self, query: str) -> str: - """Prepare a Boolean query by quoting individual terms while preserving operators. - - Args: - query: A Boolean query like "tier1-test AND unicode" or "(hello OR world) NOT test" - - Returns: - A properly formatted Boolean query with quoted terms that need quoting - """ - # Define Boolean operators and their boundaries - boolean_pattern = r"(\bAND\b|\bOR\b|\bNOT\b)" - - # Split the query by Boolean operators, keeping the operators - parts = re.split(boolean_pattern, query) - - processed_parts = [] - for part in parts: - part = part.strip() - if not part: - continue - - # If it's a Boolean operator, keep it as is - if part in ["AND", "OR", "NOT"]: - processed_parts.append(part) - else: - # Handle parentheses specially - they should be preserved for grouping - if "(" in part or ")" in part: - # Parse parenthetical expressions carefully - processed_part = self._prepare_parenthetical_term(part) - processed_parts.append(processed_part) - else: - # This is a search term - for Boolean queries, don't add prefix wildcards - prepared_term = self._prepare_single_term(part, is_prefix=False) - processed_parts.append(prepared_term) - - return " ".join(processed_parts) - - def _prepare_parenthetical_term(self, term: str) -> str: - """Prepare a term that contains parentheses, preserving the parentheses for grouping. - - Args: - term: A term that may contain parentheses like "(hello" or "world)" or "(hello OR world)" - - Returns: - A properly formatted term with parentheses preserved - """ - # Handle terms that start/end with parentheses but may contain quotable content - result = "" - i = 0 - while i < len(term): - if term[i] in "()": - # Preserve parentheses as-is - result += term[i] - i += 1 - else: - # Find the next parenthesis or end of string - start = i - while i < len(term) and term[i] not in "()": - i += 1 - - # Extract the content between parentheses - content = term[start:i].strip() - if content: - # Only quote if it actually needs quoting (has hyphens, special chars, etc) - # but don't quote if it's just simple words - if self._needs_quoting(content): - escaped_content = content.replace('"', '""') - result += f'"{escaped_content}"' - else: - result += content - - return result - - def _needs_quoting(self, term: str) -> bool: - """Check if a term needs to be quoted for FTS5 safety. - - Args: - term: The term to check - - Returns: - True if the term should be quoted - """ - if not term or not term.strip(): - return False - - # Characters that indicate we should quote (excluding parentheses which are valid syntax) - needs_quoting_chars = [ - " ", - ".", - ":", - ";", - ",", - "<", - ">", - "?", - "/", - "-", - "'", - '"', - "[", - "]", - "{", - "}", - "+", - "!", - "@", - "#", - "$", - "%", - "^", - "&", - "=", - "|", - "\\", - "~", - "`", - ] - - return any(c in term for c in needs_quoting_chars) - - def _prepare_single_term(self, term: str, is_prefix: bool = True) -> str: - """Prepare a single search term (no Boolean operators). - - Args: - term: A single search term - is_prefix: Whether to add prefix search capability (* suffix) - - Returns: - A properly formatted single term - """ - if not term or not term.strip(): - return term - - term = term.strip() - - # Check if term is already a proper wildcard pattern (alphanumeric + *) - # e.g., "hello*", "test*world" - these should be left alone - if "*" in term and all(c.isalnum() or c in "*_-" for c in term): - return term - - # Natural-language queries arrive with sentence punctuation that FTS5 - # treats as syntax ("When did Melanie paint a sunrise?"). The tokenizer - # ignores this punctuation in the INDEX, so stripping it from word - # edges loses nothing — but leaving it forces the whole question into - # an exact-phrase match that returns zero rows, silently disabling the - # FTS half of hybrid search. Interior characters (hyphens, slashes — - # permalinks and paths) are untouched. - if " " in term: - words = [word.strip("?!.,;:") for word in term.split()] - term = " ".join(word for word in words if word) - if not term: - return "" - - # Characters that can cause FTS5 syntax errors when used as operators - # We're more conservative here - only quote when we detect problematic patterns - problematic_chars = [ - '"', - "'", - "(", - ")", - "[", - "]", - "{", - "}", - "+", - "!", - "@", - "#", - "$", - "%", - "^", - "&", - "=", - "|", - "\\", - "~", - "`", - ] - - # Characters that indicate we should quote (spaces, dots, colons, etc.) - # Adding hyphens here because FTS5 can have issues with hyphens followed by wildcards - needs_quoting_chars = [" ", ".", ":", ";", ",", "<", ">", "?", "/", "-"] - - # Check if term needs quoting - has_problematic = any(c in term for c in problematic_chars) - has_spaces_or_special = any(c in term for c in needs_quoting_chars) - - if has_problematic or has_spaces_or_special: - # Handle multi-word queries differently from special character queries - if " " in term and not any(c in term for c in problematic_chars): - # Check if any individual word contains special characters that need quoting - words = term.strip().split() - has_special_in_words = any( - any(c in word for c in needs_quoting_chars if c != " ") for word in words - ) - - if not has_special_in_words: - # For multi-word queries with simple words (like "emoji unicode"), - # use boolean AND to handle word order variations - if is_prefix: - # Add prefix wildcard to each word for better matching - prepared_words = [f"{word}*" for word in words if word] - else: - prepared_words = words - term = " AND ".join(prepared_words) - else: - # If any word has special characters, quote the entire phrase - escaped_term = term.replace('"', '""') - if is_prefix and not ("/" in term and term.endswith(".md")): - term = f'"{escaped_term}"*' - else: - term = f'"{escaped_term}"' # pragma: no cover - else: - # For terms with problematic characters or file paths, use exact phrase matching - # Escape any existing quotes by doubling them - escaped_term = term.replace('"', '""') - # Quote the entire term to handle special characters safely - if is_prefix and not ("/" in term and term.endswith(".md")): - # For search terms (not file paths), add prefix matching - term = f'"{escaped_term}"*' - else: - # For file paths, use exact matching - term = f'"{escaped_term}"' - elif is_prefix: - # Only add wildcard for simple terms without special characters - term = f"{term}*" - - return term - - @override - def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: - """Prepare a search term for FTS5 query. - - Args: - term: The search term to prepare - is_prefix: Whether to add prefix search capability (* suffix) - - For FTS5: - - Boolean operators (AND, OR, NOT) are preserved for complex queries - - Terms with FTS5 special characters are quoted to prevent syntax errors - - Simple terms get prefix wildcards for better matching - """ - # Check for explicit boolean operators - if present, process as Boolean query - boolean_operators = [" AND ", " OR ", " NOT "] - if any(op in f" {term} " for op in boolean_operators): - return self._prepare_boolean_query(term) - - # For non-Boolean queries, use the single term preparation logic - return self._prepare_single_term(term, is_prefix) - - @staticmethod - def _relaxed_fts_term(word: str) -> str: - """Render one relaxed word as an FTS5-safe prefix expression. - - A word token can contain an apostrophe ("об'єкт", "don't"). Interpolated - bare it is FTS5 syntax, not text: the whole expression fails to parse, the - caller swallows the syntax error, and the relaxed retry returns nothing — - the exact silent-empty-FTS failure this fallback exists to prevent. - """ - if "'" in word or '"' in word: - return '"{}"*'.format(word.replace('"', '""')) - return f"{word}*" - - @staticmethod - def _relaxed_fts_text(search_text: Optional[str]) -> Optional[str]: - """OR-relaxed FTS5 expression for a failed strict query, or None.""" - words = relaxed_query_words(search_text) - if not words: - return None - return " OR ".join(SQLiteSearchRepository._relaxed_fts_term(word) for word in words) - @override async def semantic_effectively_enabled(self) -> bool: """Probe the sqlite-vec runtime instead of trusting still-enabled config. @@ -780,325 +497,6 @@ async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> Non # FTS search (backend-specific) # ------------------------------------------------------------------ - @staticmethod - def _is_fts5_syntax_error(exc: Exception) -> bool: - return "fts5: syntax error" in str(exc).lower() - - async def _build_fts_query_parts( - self, - search_text: Optional[str] = None, - permalink: Optional[str] = None, - permalink_match: Optional[str] = None, - title: Optional[str] = None, - note_types: Optional[List[str]] = None, - after_date: Optional[datetime] = None, - search_item_types: Optional[List[SearchItemType]] = None, - categories: Optional[List[str]] = None, - metadata_filters: Optional[dict[str, Any]] = None, - file_path_prefix: Optional[str] = None, - temporal: Optional[TemporalFilter] = None, - candidate_keys: Sequence[SearchIndexKey] | None = None, - ) -> tuple[str, str, dict[str, Any], str, str]: - """Build SQLite FTS FROM/WHERE params shared by search and count.""" - conditions = [] - match_conditions = [] - params = {} - order_by_clause = "" - from_clause = "search_index" - score_expression = "bm25(search_index)" - preserve_match_score = False - - # Handle text search for title and content - if search_text: - # Skip FTS for wildcard-only queries that would cause "unknown special query" errors - if search_text.strip() == "*" or search_text.strip() == "": - # For wildcard searches, don't add any text conditions - return all results - pass - else: - script_query = analyze_script_query(search_text.strip()) - # Trigger: the query contains text from an unsegmented script. - # Why: the script channel needs one table-level MATCH alongside word fields. - # Outcome: mixed queries rank all terms together; word-only queries retain their - # established per-column matching and ranking behavior. - if script_query.gram_phrases: - preserve_match_score = True - params["text"] = "" - params["script_text"] = "" - if script_query.word_text: - prepared_text = self._prepare_search_term(script_query.word_text) - params["text"] = ( - f"(title: ({prepared_text}) OR " - f"content_stems: ({prepared_text}) OR " - f"content_snippet: ({prepared_text}))" - ) - script_phrases = " AND ".join( - f'"{" ".join(phrase)}"' for phrase in script_query.gram_phrases - ) - script_clause = f"script_ngrams: ({script_phrases})" - params["script_text"] = ( - f" AND ({script_clause})" if script_query.word_text else script_clause - ) - match_conditions.append("search_index MATCH (:text || :script_text)") - else: - word_text = ( - script_query.word_text - if script_query.word_text is not None - else search_text.strip() - ) - processed_text = self._prepare_search_term(word_text) - params["text"] = processed_text - # content_stems is capped for Postgres index-row compatibility, while - # SQLite stores the complete note body in its FTS5 content_snippet column. - match_conditions.append( - "(search_index.title MATCH :text OR " - "search_index.content_stems MATCH :text OR " - "search_index.content_snippet MATCH :text)" - ) - - # Handle title match search - if title: - title_text = self._prepare_search_term(title.strip(), is_prefix=False) - params["title_text"] = title_text - match_conditions.append("search_index.title MATCH :title_text") - - # Handle permalink exact search - if permalink: - params["permalink"] = permalink - conditions.append("search_index.permalink = :permalink") - - # Handle permalink match search, supports * - if permalink_match: - # For GLOB patterns, don't use _prepare_search_term as it will quote slashes - # GLOB patterns need to preserve their syntax - permalink_text = permalink_match.lower().strip() - params["permalink"] = permalink_text - if "*" in permalink_match: - conditions.append("search_index.permalink GLOB :permalink") - else: - # For exact matches without *, we can use FTS5 MATCH - # but only prepare the term if it doesn't look like a path - if "/" in permalink_text: - conditions.append("search_index.permalink = :permalink") - else: - permalink_text = self._prepare_search_term(permalink_text, is_prefix=False) - params["permalink"] = permalink_text - match_conditions.append("search_index.permalink MATCH :permalink") - - # Handle directory subtree scope. The predicate is built by the shared - # helper so SQLite and Postgres scope by the identical rule; see - # file_path_prefix_condition for the boundary and escaping reasoning. - subtree_condition = file_path_prefix_condition(file_path_prefix, params) - if subtree_condition is not None: - conditions.append(subtree_condition) - - # Handle an explicit candidate-row restriction. Built by the shared helper so - # both backends restrict by the identical rule; see - # candidate_key_restriction_condition for why the vector filter pass asks about - # its candidates rather than paging the filter's whole match set (#1431). - if candidate_keys is not None: - conditions.append(candidate_key_restriction_condition(candidate_keys, params)) - - # Handle entity type filter (parameterized for defense-in-depth) - if search_item_types: - type_placeholders = [] - for idx, t in enumerate(search_item_types): - param_name = f"search_type_{idx}" - params[param_name] = t.value - type_placeholders.append(f":{param_name}") - conditions.append(f"search_index.type IN ({', '.join(type_placeholders)})") - - # Handle observation category filter (parameterized for defense-in-depth). - # Trigger: caller passed `categories` to scope observation results. - # Why: `entity_types=["observation"]` only narrows to the observation row type; - # callers expect exact-category matching, not incidental text matches. - # Outcome: only rows whose indexed category exactly equals a requested value - # survive (entities/relations have NULL category and are excluded). - if categories: - category_placeholders = [] - for idx, category in enumerate(categories): - param_name = f"category_{idx}" - params[param_name] = category - category_placeholders.append(f":{param_name}") - conditions.append(f"search_index.category IN ({', '.join(category_placeholders)})") - - # Handle note type filter (frontmatter type field, parameterized). - # Trigger: caller passed `note_types` to scope by the frontmatter `type` field. - # Why: the type belongs to the note, but only its entity row carries the - # frontmatter; observation and relation rows do not. Reading it off each row - # silently excluded every non-entity row, which made `note_types` combined - # with a valid-time filter unsatisfiable. - # Outcome: resolved through the owning note in one shared builder, so both - # backends ask the same question and observation rows of a matching note - # are admitted. - if note_types: - conditions.append( - build_note_type_predicate( - note_types, params, note_type_value=SQLITE_NOTE_TYPE_VALUE - ) - ) - - # Handle date filter using datetime() for proper comparison - if after_date: - params["after_date"] = after_date - # Filter on updated_at so recently-edited notes are included even when created_at is old - conditions.append("datetime(search_index.updated_at) > datetime(:after_date)") - - # order by most recent first - order_by_clause = ", search_index.updated_at DESC" - - # Handle authored valid time (SPEC-82). - # Trigger: caller asked when a statement was true of the world. - # Why: `after_date` above filters `updated_at`, which records when the note was - # last edited. That is bookkeeping, never a semantic claim; a decision - # effective through July says nothing about when its file was touched. - # Outcome: an independent predicate over the temporal projection. It matches - # only sources carrying a structured qualifier, so undated sources are - # excluded whenever a valid-time filter is present, and no ordering - # changes -- relevance still decides the ranking. - if temporal is not None: - conditions.append(build_temporal_predicate(temporal, params)) - - # Handle structured metadata filters (frontmatter) - if metadata_filters: - parsed_filters = parse_metadata_filters(metadata_filters) - from_clause = "search_index JOIN entity ON search_index.entity_id = entity.id" - # Frontmatter filters answer for notes only; see - # metadata_filter_content_type_condition for why every regular file - # would otherwise satisfy a null predicate. - conditions.append(metadata_filter_content_type_condition(params)) - entity_columns = await self._get_entity_columns() - - for idx, filt in enumerate(parsed_filters): - path_param = f"meta_path_{idx}" - extract_expr = None - use_tags_column = False - - if filt.path_parts == ["status"] and "frontmatter_status" in entity_columns: - extract_expr = "entity.frontmatter_status" - elif filt.path_parts == ["type"] and "frontmatter_type" in entity_columns: - extract_expr = "entity.frontmatter_type" - elif filt.path_parts == ["tags"] and "tags_json" in entity_columns: - extract_expr = "entity.tags_json" - use_tags_column = True - - if extract_expr is None: - params[path_param] = build_sqlite_json_path(filt.path_parts) - extract_expr = f"json_extract(entity.entity_metadata, :{path_param})" - - # json_extract returns SQL NULL both for a missing key and for an - # explicit JSON null, and the generated frontmatter_* columns are - # that same json_extract — so IS NULL means "the note carries no - # value here", the question `{"owner": None}` asks. `= NULL` is - # never true, so equality here would report a confident zero. - if filt.op == "is_null": - conditions.append(f"{extract_expr} IS NULL") - continue - - if filt.op == "eq": - value_param = f"meta_val_{idx}" - params[value_param] = filt.value - conditions.append(f"{extract_expr} = :{value_param}") - continue - - if filt.op == "in": - placeholders = [] - for j, val in enumerate(filt.value): - value_param = f"meta_val_{idx}_{j}" - params[value_param] = val - placeholders.append(f":{value_param}") - conditions.append(f"{extract_expr} IN ({', '.join(placeholders)})") - continue - - if filt.op == "contains": - tag_conditions = [] - for j, val in enumerate(filt.value): - value_param = f"meta_val_{idx}_{j}" - params[value_param] = val - # The exact JSON-membership test is the primary path; the - # substring patterns only reach values stored as array text. - like_condition = metadata_contains_like_condition( - extract_expr, - val, - param_prefix=value_param, - params=params, - ) - json_each_expr = ( - "json_each(entity.tags_json)" - if use_tags_column - else f"json_each(entity.entity_metadata, :{path_param})" - ) - tag_conditions.append( - "(" - f"EXISTS (SELECT 1 FROM {json_each_expr} WHERE value = :{value_param}) " - f"OR {like_condition}" - ")" - ) - conditions.append(" AND ".join(tag_conditions)) - continue - - if filt.op in {"gt", "gte", "lt", "lte", "between"}: - compare_expr = ( - f"CAST({extract_expr} AS REAL)" - if filt.comparison == "numeric" - else extract_expr - ) - - if filt.op == "between": - min_param = f"meta_val_{idx}_min" - max_param = f"meta_val_{idx}_max" - params[min_param] = filt.value[0] - params[max_param] = filt.value[1] - conditions.append(f"{compare_expr} BETWEEN :{min_param} AND :{max_param}") - else: - value_param = f"meta_val_{idx}" - params[value_param] = filt.value - operator = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[filt.op] - conditions.append(f"{compare_expr} {operator} :{value_param}") - continue - - # Trigger: SQLite rejects some Boolean combinations of MATCH predicates, - # including a word-field OR expression combined with the script channel. - # Why: each MATCH must be evaluated in an FTS-valid query context. - # Outcome: keep one outer MATCH for bm25 ranking and intersect the rest by rowid. - if len(match_conditions) > 1: - ranked_match, *additional_matches = match_conditions - conditions.extend( - f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_condition})" - for match_condition in additional_matches - ) - match_conditions = [ranked_match] - - # Trigger: SQLite FTS MATCH predicates combined with JOINs can fail with - # "unable to use function MATCH in the requested context". - # Why: script queries need MATCH and bm25 together for ranking, while legacy - # word-column OR predicates cannot evaluate bm25 in the same derived query. - # Outcome: rank script matches before joining metadata; retain the established - # rowid-filter path for word-only searches. - if metadata_filters and match_conditions: - match_where = " AND ".join(match_conditions) - if preserve_match_score: - from_clause = ( - "(SELECT search_index.rowid AS rowid, search_index.*, " - "bm25(search_index) AS fts_score " - f"FROM search_index WHERE {match_where}) AS search_index " - "JOIN entity ON search_index.entity_id = entity.id" - ) - score_expression = "search_index.fts_score" - else: - conditions.append( - f"search_index.rowid IN (SELECT rowid FROM search_index WHERE {match_where})" - ) - else: - conditions.extend(match_conditions) - - # Always filter by project_id - params["project_id"] = self.project_id - conditions.append("search_index.project_id = :project_id") - - # Build WHERE clause - where_clause = " AND ".join(conditions) if conditions else "1=1" - return from_clause, where_clause, params, order_by_clause, score_expression - @override async def search( self, @@ -1153,13 +551,11 @@ async def search( return dispatched # --- FTS mode (SQLite-specific) --- - ( - from_clause, - where_clause, - params, - order_by_clause, - score_expression, - ) = await self._build_fts_query_parts( + # Generated frontmatter columns are read only when a metadata filter needs them. + entity_columns = await self._get_entity_columns() if metadata_filters else frozenset() + compiled = compile_fts_filter( + self.scope, + entity_columns=entity_columns, search_text=search_text, permalink=permalink, permalink_match=permalink_match, @@ -1173,8 +569,7 @@ async def search( temporal=temporal, candidate_keys=candidate_keys, ) - - # set limit on search query + params = compiled.params params["limit"] = limit params["offset"] = offset relaxed_search_text = search_text @@ -1198,10 +593,10 @@ async def search( search_index.category, search_index.created_at, search_index.updated_at, - {score_expression} as score - FROM {from_clause} - WHERE {where_clause} - ORDER BY score ASC {order_by_clause} + {compiled.score_expression} as score + FROM {compiled.from_clause} + WHERE {compiled.where_clause} + ORDER BY score ASC {compiled.order_by_clause} LIMIT :limit OFFSET :offset """ @@ -1221,9 +616,7 @@ async def run_search(active_session: AsyncSession): # vector-only. # Outcome: one retry with OR-joined prefix terms; bm25 still # ranks multi-term matches first. - relaxed = ( - self._relaxed_fts_text(relaxed_search_text) if allow_relaxed and not rows else None - ) + relaxed = relaxed_fts_text(relaxed_search_text) if allow_relaxed and not rows else None if relaxed and params.get("text"): relaxed_fallback_used = True params["text"] = ( @@ -1252,7 +645,7 @@ async def run_search(active_session: AsyncSession): rows, relaxed_fallback_used = await run_search(owned_session) except Exception as e: # Handle FTS5 syntax errors and provide user-friendly feedback - if self._is_fts5_syntax_error(e): # pragma: no cover + if is_fts5_syntax_error(e): # pragma: no cover logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}") # Return empty results rather than crashing if trace is not None: @@ -1327,13 +720,10 @@ async def count( min_similarity=min_similarity, ) - ( - from_clause, - where_clause, - params, - _order_by_clause, - _score_expression, - ) = await self._build_fts_query_parts( + entity_columns = await self._get_entity_columns() if metadata_filters else frozenset() + compiled = compile_fts_filter( + self.scope, + entity_columns=entity_columns, search_text=search_text, permalink=permalink, permalink_match=permalink_match, @@ -1346,7 +736,8 @@ async def count( file_path_prefix=file_path_prefix, temporal=temporal, ) - sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" + params = compiled.params + sql = f"SELECT COUNT(*) FROM {compiled.from_clause} WHERE {compiled.where_clause}" logger.trace(f"Count {sql} params: {params}") relaxed_search_text = search_text if search_text and "script_text" in params: @@ -1356,9 +747,7 @@ async def count( result = await session.execute(text(sql), params) total = int(result.scalar_one()) relaxed = ( - self._relaxed_fts_text(relaxed_search_text) - if allow_relaxed and total == 0 - else None + relaxed_fts_text(relaxed_search_text) if allow_relaxed and total == 0 else None ) if relaxed and params.get("text"): params["text"] = ( @@ -1375,7 +764,7 @@ async def count( total = int(result.scalar_one()) return total except Exception as e: - if self._is_fts5_syntax_error(e): # pragma: no cover + if is_fts5_syntax_error(e): # pragma: no cover logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}") return 0 logger.error(f"Database error during search count: {e}") diff --git a/src/basic_memory/repository/temporal_filters.py b/src/basic_memory/repository/temporal_filters.py index 81c3c666c..d49802122 100644 --- a/src/basic_memory/repository/temporal_filters.py +++ b/src/basic_memory/repository/temporal_filters.py @@ -33,6 +33,7 @@ from typing import Any +from basic_memory.repository.search_scope import ProjectScope from basic_memory.temporal import TemporalFilter, TemporalRange TEMPORAL_INDEX_TABLE = "memory_time_index" @@ -84,7 +85,9 @@ def _not_window_ends_before_source(window: TemporalRange) -> str | None: return f"({' OR '.join(clauses)})" -def build_temporal_predicate(temporal: TemporalFilter, params: dict[str, Any]) -> str: +def build_temporal_predicate( + temporal: TemporalFilter, params: dict[str, Any], *, scope: ProjectScope +) -> str: """Build the WHERE-clause fragment restricting search rows by authored valid time. Two intervals overlap exactly when neither lies entirely before the other, which @@ -97,7 +100,8 @@ def build_temporal_predicate(temporal: TemporalFilter, params: dict[str, Any]) - documented default for a valid-time query. Binds are added to `params` in place, following the convention already used by the - surrounding FTS query builders. + surrounding FTS query builders. Assertions are read from `scope` only, and the + search row is matched on its full `(project_id, type, id)` identity. """ window = temporal.window if window is not None and window.is_empty: @@ -105,7 +109,7 @@ def build_temporal_predicate(temporal: TemporalFilter, params: dict[str, Any]) - # false constant is both correct and cheaper than running the subquery. return _MATCHES_NOTHING - conditions = [f"{TEMPORAL_INDEX_TABLE}.project_id = :project_id"] + conditions = [scope.predicate(f"{TEMPORAL_INDEX_TABLE}.project_id", params)] if temporal.kind is not None: params["tq_kind"] = temporal.kind.value @@ -135,11 +139,12 @@ def build_temporal_predicate(temporal: TemporalFilter, params: dict[str, Any]) - ) where_clause = "\n AND ".join(conditions) - # (type, id) is the search row's own identity and the address this projection - # stores, so the pair joins the two without a correlated reference. + # (project_id, type, id) is the search row's own identity and the address this + # projection stores, so the triple joins the two without a correlated reference. return ( - "(search_index.type, search_index.id) IN (\n" - f" SELECT {TEMPORAL_INDEX_TABLE}.source_type, {TEMPORAL_INDEX_TABLE}.source_id\n" + "(search_index.project_id, search_index.type, search_index.id) IN (\n" + f" SELECT {TEMPORAL_INDEX_TABLE}.project_id, " + f"{TEMPORAL_INDEX_TABLE}.source_type, {TEMPORAL_INDEX_TABLE}.source_id\n" f" FROM {TEMPORAL_INDEX_TABLE}\n" f" WHERE {where_clause})" ) diff --git a/tests/repository/test_hybrid_fusion.py b/tests/repository/test_hybrid_fusion.py index 9e49048f7..4bf7135b1 100644 --- a/tests/repository/test_hybrid_fusion.py +++ b/tests/repository/test_hybrid_fusion.py @@ -71,10 +71,6 @@ async def init_search_index(self): async def get_entity_physical_chunk_keys(self, entity_id: int) -> set[str] | None: return None # physical storage is not inspectable in this double - @override - def _prepare_search_term(self, term, is_prefix=True): - return term # pragma: no cover - @override async def search( self, diff --git a/tests/repository/test_postgres_search_quoted_queries.py b/tests/repository/test_postgres_search_quoted_queries.py index a42f709f5..40a412619 100644 --- a/tests/repository/test_postgres_search_quoted_queries.py +++ b/tests/repository/test_postgres_search_quoted_queries.py @@ -5,6 +5,7 @@ import pytest +import basic_memory.repository.postgres_search_repository as postgres_search_repository_module from basic_memory.repository.postgres_search_repository import PostgresSearchRepository from basic_memory.repository.search_index_row import SearchIndexRow @@ -60,7 +61,7 @@ async def test_quoted_or_phrases_complete_without_tsquery_recovery( ) syntax_errors: list[Exception] = [] - real_is_syntax_error = repository._is_tsquery_syntax_error + real_is_syntax_error = postgres_search_repository_module.is_tsquery_syntax_error def record_syntax_error(exception: Exception) -> bool: is_syntax_error = real_is_syntax_error(exception) @@ -68,7 +69,10 @@ def record_syntax_error(exception: Exception) -> bool: syntax_errors.append(exception) return is_syntax_error - monkeypatch.setattr(repository, "_is_tsquery_syntax_error", record_syntax_error) + # The repository module binds the classifier at import; patch it where it is read. + monkeypatch.setattr( + postgres_search_repository_module, "is_tsquery_syntax_error", record_syntax_error + ) query = '"incident response" OR "database recovery"' async with asyncio.timeout(2): diff --git a/tests/repository/test_postgres_search_repository.py b/tests/repository/test_postgres_search_repository.py index e25b785e8..308509ac7 100644 --- a/tests/repository/test_postgres_search_repository.py +++ b/tests/repository/test_postgres_search_repository.py @@ -13,6 +13,14 @@ from basic_memory.config import BasicMemoryConfig, DatabaseBackend import basic_memory.repository.search_repository_base as search_repository_base_module from basic_memory.repository.litellm_provider import LiteLLMEmbeddingProvider +import basic_memory.repository.postgres_search_query as postgres_search_query_module +import basic_memory.repository.postgres_search_repository as postgres_search_repository_module +from basic_memory.repository.postgres_search_query import ( + compile_fts_filter, + prepare_search_term, + prepare_single_term, + relaxed_tsquery_text, +) from basic_memory.repository.postgres_search_repository import ( PostgresSearchRepository, _strip_nul_from_row, @@ -261,35 +269,29 @@ async def test_postgres_search_repository_bulk_index_items_and_prepare_terms( await repo.bulk_index_items([]) # Exercise term preparation helpers - assert "&" in repo._prepare_search_term("coffee AND brewing") - assert repo._prepare_search_term("coff*") == "coff:*" - assert repo._prepare_search_term("()&!:") == "NOSPECIALCHARS:*" - assert repo._prepare_search_term("coffee brewing") == "coffee:* & brewing:*" - assert repo._prepare_single_term(" ") == " " - assert repo._prepare_single_term("coffee", is_prefix=False) == "coffee" - - indexed_from, _where, indexed_params, _order, _score = await repo._build_fts_query_parts( - search_text="coffee brewing", - allow_relaxed=True, - ) - assert "FROM search_index AS candidate_parent" in indexed_from - assert "FROM search_index_fts_chunks AS candidate_chunk" in indexed_from - assert "querytree(to_tsquery('english', :text))" in indexed_from - assert indexed_params["text_candidate"] == "coffee:* | brewing:*" - - filtered_from, _where, _params, _order, _score = await repo._build_fts_query_parts( - search_text="coffee brewing", - metadata_filters={"status": "active"}, + assert "&" in prepare_search_term("coffee AND brewing") + assert prepare_search_term("coff*") == "coff:*" + assert prepare_search_term("()&!:") == "NOSPECIALCHARS:*" + assert prepare_search_term("coffee brewing") == "coffee:* & brewing:*" + assert prepare_single_term(" ") == " " + assert prepare_single_term("coffee", is_prefix=False) == "coffee" + + indexed = compile_fts_filter(repo.scope, search_text="coffee brewing", allow_relaxed=True) + assert "FROM search_index AS candidate_parent" in indexed.from_clause + assert "FROM search_index_fts_chunks AS candidate_chunk" in indexed.from_clause + assert "querytree(to_tsquery('english', :text))" in indexed.from_clause + assert indexed.params["text_candidate"] == "coffee:* | brewing:*" + + filtered = compile_fts_filter( + repo.scope, search_text="coffee brewing", metadata_filters={"status": "active"} ) - assert "AS fts_candidate" in filtered_from - assert "JOIN entity ON search_index.entity_id = entity.id" in filtered_from + assert "AS fts_candidate" in filtered.from_clause + assert "JOIN entity ON search_index.entity_id = entity.id" in filtered.from_clause - negated_from, _where, negated_params, _order, _score = await repo._build_fts_query_parts( - search_text="coffee NOT brewing", - ) - assert "AS fts_candidate" in negated_from - assert "FROM search_index AS candidate_all" in negated_from - assert negated_params["text_candidate"] == "coffee | brewing" + negated = compile_fts_filter(repo.scope, search_text="coffee NOT brewing") + assert "AS fts_candidate" in negated.from_clause + assert "FROM search_index AS candidate_all" in negated.from_clause + assert negated.params["text_candidate"] == "coffee | brewing" now = datetime.now(timezone.utc) rows = [ @@ -374,7 +376,11 @@ async def test_postgres_search_repository_tsquery_syntax_error_returns_empty( # Isolate database-error handling from the user parser, which deliberately # repairs malformed trailing operators before they reach PostgreSQL. with monkeypatch.context() as syntax_error: - syntax_error.setattr(repo, "_prepare_search_term", lambda *_args, **_kwargs: "coffee &") + syntax_error.setattr( + postgres_search_query_module, + "prepare_search_term", + lambda *_args, **_kwargs: "coffee &", + ) results = await repo.search(search_text="coffee") assert results == [] assert await repo.count(search_text="coffee") == 0 @@ -420,8 +426,8 @@ async def test_postgres_search_tsquery_error_does_not_poison_caller_session( # without the savepoint it aborts the caller's transaction. with monkeypatch.context() as syntax_error: syntax_error.setattr( - repo, - "_prepare_search_term", + postgres_search_query_module, + "prepare_search_term", lambda *_args, **_kwargs: "coffee &", ) results = await repo.search(search_text="coffee", session=session) @@ -1193,21 +1199,21 @@ async def test_postgres_question_punctuation_and_relaxation(session_maker, test_ and a strict all-AND miss had no relaxed retry, silently disabling the FTS half of hybrid search for natural-language questions. """ - repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + PostgresSearchRepository(session_maker, project_id=test_project.id) # Edge punctuation stripped before lexeme formatting. - prepared = repo._prepare_search_term("When did Melanie paint a sunrise?") + prepared = prepare_search_term("When did Melanie paint a sunrise?") assert "?" not in prepared assert "sunrise:*" in prepared # Relaxation drops stopwords and OR-joins content terms. - relaxed = repo._relaxed_tsquery_text("When did Melanie paint a sunrise?") + relaxed = relaxed_tsquery_text("When did Melanie paint a sunrise?") assert relaxed == "melanie:* | paint:* | sunrise:*" # User intent is not second-guessed. - assert repo._relaxed_tsquery_text("alpha AND beta") is None - assert repo._relaxed_tsquery_text('"exact phrase"') is None - assert repo._relaxed_tsquery_text(None) is None + assert relaxed_tsquery_text("alpha AND beta") is None + assert relaxed_tsquery_text('"exact phrase"') is None + assert relaxed_tsquery_text(None) is None @pytest.mark.asyncio @@ -1267,7 +1273,7 @@ async def test_postgres_relaxes_after_strict_tsquery_syntax_error( ) syntax_errors: list[Exception] = [] - real_is_syntax_error = repo._is_tsquery_syntax_error + real_is_syntax_error = postgres_search_repository_module.is_tsquery_syntax_error def record_syntax_error(exc: Exception) -> bool: is_syntax_error = real_is_syntax_error(exc) @@ -1275,7 +1281,10 @@ def record_syntax_error(exc: Exception) -> bool: syntax_errors.append(exc) return is_syntax_error - monkeypatch.setattr(repo, "_is_tsquery_syntax_error", record_syntax_error) + # The repository module binds the classifier at import; patch it where it is read. + monkeypatch.setattr( + postgres_search_repository_module, "is_tsquery_syntax_error", record_syntax_error + ) query = "foo None: """Quoted user syntax must become a complete tsquery expression before SQL.""" - assert _make_repo()._prepare_search_term(query) == expected + assert prepare_search_term(query) == expected def test_postgres_many_quoted_groups_restore_atomically() -> None: @@ -632,4 +633,4 @@ def test_postgres_many_quoted_groups_restore_atomically() -> None: query = " OR ".join(f'"term{index} word{index}"' for index in range(11)) expected = " | ".join(f"(term{index} & word{index})" for index in range(11)) - assert _make_repo()._prepare_search_term(query) == expected + assert prepare_search_term(query) == expected diff --git a/tests/repository/test_search_file_path_prefix.py b/tests/repository/test_search_file_path_prefix.py index 885cd96ba..c6d2b3954 100644 --- a/tests/repository/test_search_file_path_prefix.py +++ b/tests/repository/test_search_file_path_prefix.py @@ -335,7 +335,7 @@ async def test_semantic_retrieval_honors_the_scope( def test_condition_is_one_shared_predicate_for_both_dialects(): """The SQL text and its parameters are backend-independent by construction. - Both `_build_fts_query_parts` implementations call this one helper, so the + Both `compile_fts_filter` implementations call this one helper, so the identical-behavior claim above is structural rather than a coincidence two hand-written predicates happen to share. """ diff --git a/tests/repository/test_search_relaxed_rendering.py b/tests/repository/test_search_relaxed_rendering.py index fc43d062c..93188ef9a 100644 --- a/tests/repository/test_search_relaxed_rendering.py +++ b/tests/repository/test_search_relaxed_rendering.py @@ -4,8 +4,8 @@ import pytest -from basic_memory.repository.postgres_search_repository import PostgresSearchRepository -from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository +from basic_memory.repository.postgres_search_query import relaxed_tsquery_text +from basic_memory.repository.sqlite_search_query import relaxed_fts_text CREATE_FTS = ( "CREATE VIRTUAL TABLE t USING fts5(" @@ -27,7 +27,7 @@ ) def test_sqlite_relaxed_text_quotes_only_terms_that_need_it(query: str, expected: str) -> None: """Apostrophe terms are quoted; every other term renders exactly as before.""" - assert SQLiteSearchRepository._relaxed_fts_text(query) == expected + assert relaxed_fts_text(query) == expected @pytest.mark.parametrize( @@ -47,7 +47,7 @@ def test_sqlite_relaxed_text_is_accepted_by_fts5(query: str) -> None: catches and turns into an empty result — the relaxed retry then silently contributes nothing, which is the failure this fallback exists to prevent. """ - relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + relaxed = relaxed_fts_text(query) assert relaxed is not None connection = sqlite3.connect(":memory:") @@ -82,7 +82,7 @@ def test_sqlite_relaxed_text_bare_apostrophe_would_be_rejected() -> None: ) def test_postgres_relaxed_tsquery_quotes_apostrophe_lexemes(query: str, expected: str) -> None: """Postgres carries the same token shapes, so it needs the same escaping.""" - assert PostgresSearchRepository._relaxed_tsquery_text(query) == expected + assert relaxed_tsquery_text(query) == expected @pytest.mark.parametrize( @@ -105,7 +105,7 @@ def test_relaxed_terms_match_the_stored_note(document: str, query: str) -> None: try: connection.execute(CREATE_FTS) connection.execute("INSERT INTO t VALUES (?)", (document,)) - relaxed = SQLiteSearchRepository._relaxed_fts_text(query) + relaxed = relaxed_fts_text(query) assert relaxed is not None rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() finally: @@ -125,7 +125,7 @@ def test_relaxed_terms_match_either_stored_form(document: str) -> None: try: connection.execute(CREATE_FTS) connection.execute("INSERT INTO t VALUES (?)", (document,)) - relaxed = SQLiteSearchRepository._relaxed_fts_text("foo­bar права доступа") + relaxed = relaxed_fts_text("foo­bar права доступа") assert relaxed is not None rows = connection.execute("SELECT rowid FROM t WHERE t MATCH ?", (relaxed,)).fetchall() finally: @@ -138,5 +138,5 @@ def test_orthographic_joiners_are_not_duplicated_into_a_second_variant() -> None A stripped variant would only widen the OR with a term no note can hold. """ - relaxed = SQLiteSearchRepository._relaxed_fts_text("نمی‌خواهم دسترسی را لغو") + relaxed = relaxed_fts_text("نمی‌خواهم دسترسی را لغو") assert relaxed == "نمی‌خواهم* OR دسترسی* OR را* OR لغو*" diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index c73cd02c0..11b95f4e0 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -10,6 +10,7 @@ from basic_memory.models import Entity from basic_memory.models.project import Project from basic_memory.repository.search_repository import SearchIndexRow +from basic_memory.repository import postgres_search_query, sqlite_search_query from basic_memory.repository.postgres_search_repository import PostgresSearchRepository from basic_memory.schemas.search import SearchItemType @@ -19,6 +20,13 @@ def is_postgres_backend(search_repository): return isinstance(search_repository, PostgresSearchRepository) +def fts_query(search_repository): + """The term-preparation module for the repository's backend.""" + if is_postgres_backend(search_repository): + return postgres_search_query + return sqlite_search_query + + @pytest_asyncio.fixture async def search_entity(session_maker, test_project: Project): """Create a test entity for search testing.""" @@ -616,53 +624,57 @@ class TestSearchTermPreparation: def test_simple_terms_get_prefix_wildcard(self, search_repository): """Simple alphanumeric terms should get prefix matching.""" - from basic_memory.repository.postgres_search_repository import PostgresSearchRepository - - if isinstance(search_repository, PostgresSearchRepository): + if is_postgres_backend(search_repository): # Postgres tsquery uses :* for prefix matching - assert search_repository._prepare_search_term("hello") == "hello:*" - assert search_repository._prepare_search_term("project") == "project:*" - assert search_repository._prepare_search_term("test123") == "test123:*" + assert fts_query(search_repository).prepare_search_term("hello") == "hello:*" + assert fts_query(search_repository).prepare_search_term("project") == "project:*" + assert fts_query(search_repository).prepare_search_term("test123") == "test123:*" else: # SQLite FTS5 uses * for prefix matching - assert search_repository._prepare_search_term("hello") == "hello*" - assert search_repository._prepare_search_term("project") == "project*" - assert search_repository._prepare_search_term("test123") == "test123*" + assert fts_query(search_repository).prepare_search_term("hello") == "hello*" + assert fts_query(search_repository).prepare_search_term("project") == "project*" + assert fts_query(search_repository).prepare_search_term("test123") == "test123*" def test_terms_with_existing_wildcard_unchanged(self, search_repository): """Terms that already contain * should remain unchanged.""" if is_postgres_backend(search_repository): # Postgres uses different syntax (:* instead of *) - assert search_repository._prepare_search_term("hello*") == "hello:*" - assert search_repository._prepare_search_term("test*world") == "test:*world" + assert fts_query(search_repository).prepare_search_term("hello*") == "hello:*" + assert fts_query(search_repository).prepare_search_term("test*world") == "test:*world" else: - assert search_repository._prepare_search_term("hello*") == "hello*" - assert search_repository._prepare_search_term("test*world") == "test*world" + assert fts_query(search_repository).prepare_search_term("hello*") == "hello*" + assert fts_query(search_repository).prepare_search_term("test*world") == "test*world" def test_boolean_operators_preserved(self, search_repository): """Boolean operators should be preserved without modification.""" if is_postgres_backend(search_repository): # Postgres converts AND/OR/NOT to &/|/! - assert search_repository._prepare_search_term("hello AND world") == "hello & world" - assert search_repository._prepare_search_term("cat OR dog") == "cat | dog" + assert ( + fts_query(search_repository).prepare_search_term("hello AND world") + == "hello & world" + ) + assert fts_query(search_repository).prepare_search_term("cat OR dog") == "cat | dog" # NOT must be converted to "& !" for proper tsquery syntax assert ( - search_repository._prepare_search_term("project NOT meeting") + fts_query(search_repository).prepare_search_term("project NOT meeting") == "project & !meeting" ) assert ( - search_repository._prepare_search_term("(hello AND world) OR test") + fts_query(search_repository).prepare_search_term("(hello AND world) OR test") == "(hello & world) | test" ) else: - assert search_repository._prepare_search_term("hello AND world") == "hello AND world" - assert search_repository._prepare_search_term("cat OR dog") == "cat OR dog" assert ( - search_repository._prepare_search_term("project NOT meeting") + fts_query(search_repository).prepare_search_term("hello AND world") + == "hello AND world" + ) + assert fts_query(search_repository).prepare_search_term("cat OR dog") == "cat OR dog" + assert ( + fts_query(search_repository).prepare_search_term("project NOT meeting") == "project NOT meeting" ) assert ( - search_repository._prepare_search_term("(hello AND world) OR test") + fts_query(search_repository).prepare_search_term("(hello AND world) OR test") == "(hello AND world) OR test" ) @@ -672,30 +684,30 @@ def test_hyphenated_terms_with_boolean_operators(self, search_repository): pytest.skip("This test is for SQLite FTS5-specific quoting behavior") # Test the specific case from the GitHub issue - result = search_repository._prepare_search_term("tier1-test AND unicode") + result = fts_query(search_repository).prepare_search_term("tier1-test AND unicode") assert result == '"tier1-test" AND unicode' # Test other hyphenated Boolean combinations assert ( - search_repository._prepare_search_term("multi-word OR single") + fts_query(search_repository).prepare_search_term("multi-word OR single") == '"multi-word" OR single' ) assert ( - search_repository._prepare_search_term("well-formed NOT badly-formed") + fts_query(search_repository).prepare_search_term("well-formed NOT badly-formed") == '"well-formed" NOT "badly-formed"' ) assert ( - search_repository._prepare_search_term("test-case AND (hello OR world)") + fts_query(search_repository).prepare_search_term("test-case AND (hello OR world)") == '"test-case" AND (hello OR world)' ) # Test mixed special characters with Boolean operators assert ( - search_repository._prepare_search_term("config.json AND test-file") + fts_query(search_repository).prepare_search_term("config.json AND test-file") == '"config.json" AND "test-file"' ) assert ( - search_repository._prepare_search_term("C++ OR python-script") + fts_query(search_repository).prepare_search_term("C++ OR python-script") == '"C++" OR "python-script"' ) @@ -705,11 +717,14 @@ def test_programming_terms_should_work(self, search_repository): pytest.skip("This test is for SQLite FTS5-specific behavior") # These should be quoted to handle special characters safely - assert search_repository._prepare_search_term("C++") == '"C++"*' - assert search_repository._prepare_search_term("function()") == '"function()"*' - assert search_repository._prepare_search_term("email@domain.com") == '"email@domain.com"*' - assert search_repository._prepare_search_term("array[index]") == '"array[index]"*' - assert search_repository._prepare_search_term("config.json") == '"config.json"*' + assert fts_query(search_repository).prepare_search_term("C++") == '"C++"*' + assert fts_query(search_repository).prepare_search_term("function()") == '"function()"*' + assert ( + fts_query(search_repository).prepare_search_term("email@domain.com") + == '"email@domain.com"*' + ) + assert fts_query(search_repository).prepare_search_term("array[index]") == '"array[index]"*' + assert fts_query(search_repository).prepare_search_term("config.json") == '"config.json"*' def test_malformed_fts5_syntax_quoted(self, search_repository): """Malformed FTS5 syntax should be quoted to prevent errors.""" @@ -717,17 +732,21 @@ def test_malformed_fts5_syntax_quoted(self, search_repository): pytest.skip("This test is for SQLite FTS5-specific behavior") # Multiple operators without proper syntax - assert search_repository._prepare_search_term("+++invalid+++") == '"+++invalid+++"*' - assert search_repository._prepare_search_term("!!!error!!!") == '"!!!error!!!"*' - assert search_repository._prepare_search_term("@#$%^&*()") == '"@#$%^&*()"*' + assert ( + fts_query(search_repository).prepare_search_term("+++invalid+++") == '"+++invalid+++"*' + ) + assert fts_query(search_repository).prepare_search_term("!!!error!!!") == '"!!!error!!!"*' + assert fts_query(search_repository).prepare_search_term("@#$%^&*()") == '"@#$%^&*()"*' def test_quoted_strings_handled_properly(self, search_repository): """Strings with quotes should have quotes escaped.""" if is_postgres_backend(search_repository): pytest.skip("This test is for SQLite FTS5-specific behavior") - assert search_repository._prepare_search_term('say "hello"') == '"say ""hello"""*' - assert search_repository._prepare_search_term("it's working") == '"it\'s working"*' + assert fts_query(search_repository).prepare_search_term('say "hello"') == '"say ""hello"""*' + assert ( + fts_query(search_repository).prepare_search_term("it's working") == '"it\'s working"*' + ) def test_file_paths_no_prefix_wildcard(self, search_repository): """File paths should not get prefix wildcards.""" @@ -735,11 +754,11 @@ def test_file_paths_no_prefix_wildcard(self, search_repository): pytest.skip("This test is for SQLite FTS5-specific behavior") assert ( - search_repository._prepare_search_term("config.json", is_prefix=False) + fts_query(search_repository).prepare_search_term("config.json", is_prefix=False) == '"config.json"' ) assert ( - search_repository._prepare_search_term("docs/readme.md", is_prefix=False) + fts_query(search_repository).prepare_search_term("docs/readme.md", is_prefix=False) == '"docs/readme.md"' ) @@ -748,9 +767,12 @@ def test_spaces_handled_correctly(self, search_repository): if is_postgres_backend(search_repository): pytest.skip("This test is for SQLite FTS5-specific behavior") - assert search_repository._prepare_search_term("hello world") == "hello* AND world*" assert ( - search_repository._prepare_search_term("project planning") == "project* AND planning*" + fts_query(search_repository).prepare_search_term("hello world") == "hello* AND world*" + ) + assert ( + fts_query(search_repository).prepare_search_term("project planning") + == "project* AND planning*" ) def test_version_strings_with_dots_handled_correctly(self, search_repository): @@ -760,7 +782,7 @@ def test_version_strings_with_dots_handled_correctly(self, search_repository): # This reproduces the bug where "Basic Memory v0.13.0b2" becomes "Basic* AND Memory* AND v0.13.0b2*" # which causes FTS5 syntax errors because v0.13.0b2* is not valid FTS5 syntax - result = search_repository._prepare_search_term("Basic Memory v0.13.0b2") + result = fts_query(search_repository).prepare_search_term("Basic Memory v0.13.0b2") # Should be quoted because of dots in v0.13.0b2 assert result == '"Basic Memory v0.13.0b2"*' @@ -770,12 +792,18 @@ def test_mixed_special_characters_in_multi_word_queries(self, search_repository) pytest.skip("This test is for SQLite FTS5-specific behavior") # Any word containing special characters should cause the entire phrase to be quoted - assert search_repository._prepare_search_term("config.json file") == '"config.json file"*' assert ( - search_repository._prepare_search_term("user@email.com account") + fts_query(search_repository).prepare_search_term("config.json file") + == '"config.json file"*' + ) + assert ( + fts_query(search_repository).prepare_search_term("user@email.com account") == '"user@email.com account"*' ) - assert search_repository._prepare_search_term("node.js and react") == '"node.js and react"*' + assert ( + fts_query(search_repository).prepare_search_term("node.js and react") + == '"node.js and react"*' + ) @pytest.mark.asyncio async def test_search_with_special_characters_returns_results(self, search_repository): @@ -913,15 +941,19 @@ async def test_wildcard_only_search(self, search_repository, search_entity): def test_boolean_query_empty_parts_coverage(self, search_repository): """Test Boolean query parsing with empty parts (line 143 coverage).""" # Create queries that will result in empty parts after splitting - result1 = search_repository._prepare_boolean_query( + result1 = fts_query(search_repository).prepare_boolean_query( "hello AND AND world" ) # Double operator assert "hello" in result1 and "world" in result1 - result2 = search_repository._prepare_boolean_query(" OR test") # Leading operator + result2 = fts_query(search_repository).prepare_boolean_query( + " OR test" + ) # Leading operator assert "test" in result2 - result3 = search_repository._prepare_boolean_query("test OR ") # Trailing operator + result3 = fts_query(search_repository).prepare_boolean_query( + "test OR " + ) # Trailing operator assert "test" in result3 def test_parenthetical_term_quote_escaping(self, search_repository): @@ -930,12 +962,12 @@ def test_parenthetical_term_quote_escaping(self, search_repository): pytest.skip("This test is for SQLite FTS5-specific behavior") # Test term with quotes that needs escaping - result = search_repository._prepare_parenthetical_term('(say "hello" world)') + result = fts_query(search_repository).prepare_parenthetical_term('(say "hello" world)') # Should escape quotes by doubling them assert '""hello""' in result # Test term with single quotes - result2 = search_repository._prepare_parenthetical_term("(it's working)") + result2 = fts_query(search_repository).prepare_parenthetical_term("(it's working)") assert "it's working" in result2 def test_needs_quoting_empty_input(self, search_repository): @@ -944,26 +976,26 @@ def test_needs_quoting_empty_input(self, search_repository): pytest.skip("This test is for SQLite FTS5-specific behavior") # Test empty string - assert not search_repository._needs_quoting("") + assert not fts_query(search_repository).needs_quoting("") # Test whitespace-only string - assert not search_repository._needs_quoting(" ") + assert not fts_query(search_repository).needs_quoting(" ") # Test None-like cases - assert not search_repository._needs_quoting("\t") + assert not fts_query(search_repository).needs_quoting("\t") def test_prepare_single_term_empty_input(self, search_repository): """Test _prepare_single_term with empty inputs (line 227 coverage).""" # Test empty string - result1 = search_repository._prepare_single_term("") + result1 = fts_query(search_repository).prepare_single_term("") assert result1 == "" # Test whitespace-only string - result2 = search_repository._prepare_single_term(" ") + result2 = fts_query(search_repository).prepare_single_term(" ") assert result2 == " " # Should return as-is # Test string that becomes empty after strip - result3 = search_repository._prepare_single_term("\t\n") + result3 = fts_query(search_repository).prepare_single_term("\t\n") assert result3 == "\t\n" # Should return original @@ -1192,7 +1224,7 @@ async def test_question_punctuation_does_not_phrase_quote(search_repository): '"When did Melanie paint a sunrise?"*' — zero rows for any corpus — which silently disabled the FTS half of hybrid search for question queries. """ - prepared = search_repository._prepare_single_term("When did Melanie paint a sunrise?") + prepared = fts_query(search_repository).prepare_single_term("When did Melanie paint a sunrise?") assert '"' not in prepared # Prefix syntax differs by backend: FTS5 uses '*', tsquery uses ':*'. if is_postgres_backend(search_repository): @@ -1205,10 +1237,10 @@ async def test_question_punctuation_does_not_phrase_quote(search_repository): async def test_relaxed_query_drops_stopwords(search_repository): """Relaxation keys on content-bearing terms in each backend's syntax.""" if is_postgres_backend(search_repository): - relaxed = search_repository._relaxed_tsquery_text("When did Melanie paint a sunrise?") + relaxed = postgres_search_query.relaxed_tsquery_text("When did Melanie paint a sunrise?") assert relaxed == "melanie:* | paint:* | sunrise:*" else: - relaxed = search_repository._relaxed_fts_text("When did Melanie paint a sunrise?") + relaxed = sqlite_search_query.relaxed_fts_text("When did Melanie paint a sunrise?") assert relaxed == "melanie* OR paint* OR sunrise*" @@ -1216,14 +1248,14 @@ async def test_relaxed_query_drops_stopwords(search_repository): async def test_relaxed_query_preserves_punctuated_ascii_token_pieces(search_repository): """Hyphenated and slashed ASCII terms should relax using their regex token pieces.""" if is_postgres_backend(search_repository): - relaxed = search_repository._relaxed_tsquery_text("client-side state management") + relaxed = postgres_search_query.relaxed_tsquery_text("client-side state management") assert relaxed == "client:* | side:* | state:* | management:*" - slashed = search_repository._relaxed_tsquery_text("foo/bar baz qux") + slashed = postgres_search_query.relaxed_tsquery_text("foo/bar baz qux") assert slashed == "foo:* | bar:* | baz:* | qux:*" else: - relaxed = search_repository._relaxed_fts_text("client-side state management") + relaxed = sqlite_search_query.relaxed_fts_text("client-side state management") assert relaxed == "client* OR side* OR state* OR management*" - slashed = search_repository._relaxed_fts_text("foo/bar baz qux") + slashed = sqlite_search_query.relaxed_fts_text("foo/bar baz qux") assert slashed == "foo* OR bar* OR baz* OR qux*" @@ -1231,10 +1263,10 @@ async def test_relaxed_query_preserves_punctuated_ascii_token_pieces(search_repo async def test_relaxed_query_supports_whitespace_separated_cjk_terms(search_repository): """CJK terms separated by spaces should relax even when ASCII tokenization finds none.""" if is_postgres_backend(search_repository): - relaxed = search_repository._relaxed_tsquery_text("季度 报告") + relaxed = postgres_search_query.relaxed_tsquery_text("季度 报告") assert relaxed == "季度:* | 报告:*" else: - relaxed = search_repository._relaxed_fts_text("季度 报告") + relaxed = sqlite_search_query.relaxed_fts_text("季度 报告") assert relaxed == "季度* OR 报告*" @@ -1243,9 +1275,9 @@ async def test_relaxed_query_respects_user_intent(search_repository): # Eligibility matches the service-level relaxation (both backends): quoted, # boolean, short (<3 tokens), and numeric-identifier queries are not relaxed. relaxer = ( - search_repository._relaxed_tsquery_text + postgres_search_query.relaxed_tsquery_text if is_postgres_backend(search_repository) - else search_repository._relaxed_fts_text + else sqlite_search_query.relaxed_fts_text ) assert relaxer("alpha AND beta") is None assert relaxer('"exact phrase"') is None diff --git a/tests/repository/test_search_scope.py b/tests/repository/test_search_scope.py new file mode 100644 index 000000000..8120614a0 --- /dev/null +++ b/tests/repository/test_search_scope.py @@ -0,0 +1,47 @@ +"""ProjectScope: the explicit project set every search statement binds.""" + +from typing import Any, cast + +import pytest + +from basic_memory.repository.search_scope import ProjectScope + + +def test_of_sorts_and_dedupes() -> None: + assert ProjectScope.of([3, 1, 3, 2]).project_ids == (1, 2, 3) + assert ProjectScope.of([3, 1, 3, 2]) == ProjectScope.of((1, 2, 3)) + + +def test_single_and_empty() -> None: + assert ProjectScope.single(7).project_ids == (7,) + assert not ProjectScope.single(7).is_empty + assert ProjectScope.of([]).is_empty + + +@pytest.mark.parametrize("bad", [0, -1, True]) +def test_rejects_non_positive_ids(bad: int) -> None: + with pytest.raises(ValueError, match="positive integers"): + ProjectScope.of([bad]) + + +def test_rejects_non_int_ids() -> None: + with pytest.raises(ValueError, match="positive integers"): + ProjectScope.of(cast("list[int]", ["1"])) + + +def test_predicate_binds_each_id_once_per_statement() -> None: + params: dict[str, Any] = {} + scope = ProjectScope.of([5, 2]) + assert ( + scope.predicate("search_index.project_id", params) + == "search_index.project_id IN (:scope_0, :scope_1)" + ) + # A second reference within the same statement reuses the binds. + assert scope.predicate("owner.project_id", params) == "owner.project_id IN (:scope_0, :scope_1)" + assert params == {"scope_0": 2, "scope_1": 5} + + +def test_empty_scope_matches_nothing_and_binds_nothing() -> None: + params: dict[str, Any] = {} + assert ProjectScope.of([]).predicate("search_index.project_id", params) == "1 = 0" + assert params == {} diff --git a/tests/repository/test_semantic_search_base.py b/tests/repository/test_semantic_search_base.py index 6e08fe31c..11a9030e1 100644 --- a/tests/repository/test_semantic_search_base.py +++ b/tests/repository/test_semantic_search_base.py @@ -80,10 +80,6 @@ async def record_entity_vector_deferrals( return None # no session_maker in this double; the real write is covered # in tests/services/test_project_readiness.py - @override - def _prepare_search_term(self, term, is_prefix=True): - return term - @override async def search( self, diff --git a/tests/repository/test_semantic_vector_sync.py b/tests/repository/test_semantic_vector_sync.py index 307b53030..a1669e464 100644 --- a/tests/repository/test_semantic_vector_sync.py +++ b/tests/repository/test_semantic_vector_sync.py @@ -45,10 +45,6 @@ async def init_search_index(self): async def get_entity_physical_chunk_keys(self, entity_id: int) -> set[str] | None: return None # physical storage is not inspectable in this double - @override - def _prepare_search_term(self, term, is_prefix=True): - return term - @override async def search( self, diff --git a/tests/repository/test_vector_pagination.py b/tests/repository/test_vector_pagination.py index 9139f4859..a75a7ce99 100644 --- a/tests/repository/test_vector_pagination.py +++ b/tests/repository/test_vector_pagination.py @@ -55,10 +55,6 @@ async def init_search_index(self): async def get_entity_physical_chunk_keys(self, entity_id: int) -> set[str] | None: return None # physical storage is not inspectable in this double - @override - def _prepare_search_term(self, term, is_prefix=True): - return term # pragma: no cover - @override async def search( self, diff --git a/tests/repository/test_vector_threshold.py b/tests/repository/test_vector_threshold.py index 1c6323aac..04bcf2c58 100644 --- a/tests/repository/test_vector_threshold.py +++ b/tests/repository/test_vector_threshold.py @@ -57,10 +57,6 @@ async def init_search_index(self): async def get_entity_physical_chunk_keys(self, entity_id: int) -> set[str] | None: return None # physical storage is not inspectable in this double - @override - def _prepare_search_term(self, term, is_prefix=True): - return term # pragma: no cover - @override async def search( self,