Support for embedding models via Ollama and OpenAI-like endpoints

AGENTS.md

@@ -118,7 +118,7 @@ Invocation (the plain `docker compose` command uses `docker-compose.yml`):
 - **Extensions**: Installed under `config/extensions/`. Each extension can include its own `requirements.txt` (auto-installed by `setup.py`). Enabled/disabled via `extensions.enabled` setting.
 
 ## Versioning and Migrations
-- The current version is stored in the `VERSION` file (currently `1.53`). **Do not edit this file casually.**
+- The current version is stored in the `VERSION` file. **Do not edit this file casually.**
 - `config/.current-version` tracks the last-migrated version for the running instance.
 - `helper-scripts/migrate.py` compares `VERSION` to `.current-version` and runs the appropriate `migrate-X.XX-X.XX.py` scripts from `helper-scripts/migrate/` in sequence.
 - Migration scripts handle database schema changes, data transformations, and config updates between versions.

VERSION

@@ -1,4 +1,4 @@
-1.57
+1.58
 
 This file should not be modified. It is used by 4CAT to determine whether it
 needs to run migration scripts to e.g. update the database structure to a more

common/assets/llms.json

@@ -1,4 +1,25 @@
 [
+  {
+    "model": "gpt-5.6-sol",
+    "name": "[OpenAI] GPT-5.6 Sol",
+    "model_card": "https://developers.openai.com/api/docs/models/gpt-5.6-sol",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.6-terra",
+    "name": "[OpenAI] GPT-5.6 Terra",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.6-terra",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.6-luna",
+    "name": "[OpenAI] GPT-5.6 Luna",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.6-luna",
+    "provider": "openai"
+  },{
+    "model": "gpt-5.5",
+    "name": "[OpenAI] GPT-5.5",
+    "model_card": "https://platform.openai.com/docs/models/gpt-5.5",
+    "provider": "openai"
+  },
   {
     "model": "gpt-5.5",
     "name": "[OpenAI] GPT-5.5",
@@ -48,44 +69,32 @@
     "provider": "google"
   },
   {
-    "model": "gemini-3.5-flash",
-    "name": "[Google] Gemini 3.5 Flash",
-    "model_card": "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash",
+    "model": "gemini-3.8-flash",
+    "name": "[Google] Gemini 3.8 Flash",
+    "model_card": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash",
     "provider": "google"
   },
   {
-    "model": "gemini-3-flash-preview",
-    "name": "[Google] Gemini 3 Flash",
-    "model_card": "https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-flash",
+    "model": "gemini-3.5-flash-lite-preview",
+    "name": "[Google] Gemini 3.5 Flash Lite",
+    "model_card": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-5-flash-lite",
     "provider": "google"
   },
   {
-    "model": "gemini-3.1-flash-lite-preview",
-    "name": "[Google] Gemini 3.1 Flash Lite",
-    "provider": "google",
-    "model_card": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-1-flash-lite"
-  },
-  {
-    "model": "claude-fable-5",
-    "name": "[Anthropic] Claude Fable 5",
+    "model": "claude-fable-5-1",
+    "name": "[Anthropic] Claude Fable 5.1",
     "model_card": "https://www.anthropic.com/claude/fable",
     "provider": "anthropic"
   },
   {
-    "model": "claude-opus-4-8",
-    "name": "[Anthropic] Claude Opus 4.8",
-    "model_card": "https://www.anthropic.com/claude/opus",
-    "provider": "anthropic"
-  },
-  {
-    "model": "claude-opus-4-6",
-    "name": "[Anthropic] Claude Opus 4.6",
+    "model": "claude-opus-5",
+    "name": "[Anthropic] Claude Opus 5",
     "model_card": "https://www.anthropic.com/claude/opus",
     "provider": "anthropic"
   },
   {
-    "model": "claude-sonnet-4-6",
-    "name": "[Anthropic] Claude Sonnet 4.6 (latest)",
+    "model": "claude-sonnet-5",
+    "name": "[Anthropic] Claude Sonnet 5 (latest)",
     "model_card": "https://www.anthropic.com/claude/sonnet",
     "provider": "anthropic"
   },

common/lib/config_definition.py

@@ -636,13 +636,13 @@
                 "type": UserInput.OPTION_TEXT,
                 "help": "Authentication Header",
                 "default": "",
-                "tooltip": "The HTTP header used to authenticate with the server (e.g. 'X-API-KEY', 'Authorization'). Passed with the Authentication Key as value.",
+                "tooltip": "The HTTP header used to authenticate with the server (e.g. 'X-API-KEY', 'Authorization'). Passed with the Authentication Key as value. For a bearer token you can write either 'Authorization' with 'Bearer <key>' as the value, or 'Authorization: Bearer' with just the key.",
             },
             "auth_key": {
                 "type": UserInput.OPTION_TEXT,
                 "default": "",
                 "help": "Authentication Key",
-                "tooltip": "The API key to access the LLM server, if required.",
+                "tooltip": "The API key to access the LLM server, if required. 'Bearer <key>' for bearer tokens.",
             },
         }
     },

common/lib/embeddings.py

@@ -0,0 +1,121 @@
+"""
+Helpers for processors that need all vectors of an embeddings dataset at once.
+Mostly to reduce memory errors
+
+"""
+import numpy as np
+
+from common.lib.exceptions import ProcessorInterruptedException
+
+# How often to report progress while reading, in items
+STATUS_EVERY = 1000
+
+
+def get_vector(record: dict) -> list:
+    """
+    Get the embedding of a record as a list of numbers.
+
+    :param dict record:  Embedding record
+    :return list:  The vector, or an empty list if the record has none
+    """
+    vector = record.get("embedding")
+    if isinstance(vector, str):
+        vector = [float(value) for value in vector.split() if value]
+    return vector or []
+
+
+def read_embedding_matrix(dataset, limit: int, processor=None, accept=None, keep=None) -> tuple:
+    """
+    Read up to `limit` embeddings into one float32 array.
+
+    The array is allocated once, at its full size, instead of collecting rows
+    and stacking them, which would hold every vector twice for a moment.
+
+    :param dataset:  Embeddings dataset
+    :param int limit:  Most embeddings to read
+    :param processor:  Processor reading it, for progress and interruption
+    :param accept:  Optional `function(record) -> bool`; records it rejects
+      are skipped
+    :param keep:  Optional `function(record) -> anything`, to keep a small
+      summary of each record read (never the vector itself)
+    :return tuple:  `(matrix, positions, kept)`: one row per embedding read;
+      the position of each among the items `iterate_items()` yields, to find
+      them again in a second pass; and what `keep` returned for each (empty
+      without `keep`)
+    :raises ValueError:  If the embeddings are not all the same size
+    """
+    matrix = None
+    positions, kept = [], []
+
+    for position, item in enumerate(dataset.iterate_items(processor, get_annotations=False)):
+        if processor is not None and processor.interrupted:
+            raise ProcessorInterruptedException("Interrupted while reading embeddings")
+
+        if len(positions) >= limit:
+            break
+
+        record = item.original
+
+        if accept is not None and not accept(record):
+            continue
+
+        vector = get_vector(record)
+        if not vector:
+            continue
+
+        if matrix is None:
+            matrix = np.empty((limit, len(vector)), dtype=np.float32)
+        elif len(vector) != matrix.shape[1]:
+            # a ragged matrix cannot be compared; this means the dataset mixes
+            # models, which should not happen but is worth saying out loud
+            raise ValueError("The embeddings are not all the same size.")
+
+        matrix[len(positions)] = vector
+        positions.append(position)
+        if keep is not None:
+            kept.append(keep(record))
+
+        if processor is not None and len(positions) % STATUS_EVERY == 0:
+            processor.dataset.update_status(f"Read {len(positions):,} embeddings")
+
+    if matrix is None:
+        return np.empty((0, 0), dtype=np.float32), [], []
+
+    return matrix[:len(positions)], positions, kept
+
+
+def find_embedding(dataset, match, processor=None):
+    """
+    Find the first record that matches, and its vector.
+
+    :param dataset:  Embeddings dataset
+    :param match:  `function(record) -> bool`
+    :param processor:  Processor reading it, to stop when it is interrupted
+    :return tuple:  `(vector as float32 array, record)`, or `(None, None)`
+    """
+    for item in dataset.iterate_items(processor, get_annotations=False):
+        if processor is not None and processor.interrupted:
+            raise ProcessorInterruptedException("Interrupted while reading embeddings")
+
+        record = item.original
+        if match(record):
+            vector = get_vector(record)
+            if vector:
+                return np.asarray(vector, dtype=np.float32), record
+    return None, None
+
+
+def normalise_in_place(matrix):
+    """
+    Scale each row to length 1, without making a copy.
+
+    Embeddings are compared by angle (cosine similarity), not length. On unit
+    vectors, Euclidean distances and dot products follow the angles.
+
+    :param matrix:  2D float array, one embedding per row; changed in place
+    :return:  The same array
+    """
+    norms = np.linalg.norm(matrix, axis=1, keepdims=True)
+    norms[norms == 0] = 1
+    matrix /= norms
+    return matrix

common/lib/exceptions.py

@@ -162,3 +162,9 @@ class MediaSignatureException(FourcatException):
     Raise in media processors when the media cannot be read
     """
     pass
+
+class LLMServerException(FourcatException):
+    """
+    Raise when an LLM server request fails or returns something unusable
+    """
+    pass