silently dropped record_memory calls now warn once

EVALUATIONS.md

@@ -414,7 +414,7 @@ scorer = (
     client.monitor.judge_scorers
     .builder(
         name="Strict grading",
-        judge_model="claude-opus-4-8",                    # any model id your judge key can reach (list_models() on the hosted platform)
+        judge_model="claude-opus-5",                    # any model id your judge key can reach (list_models() on the hosted platform)
         judge_prompt="""You are grading a customer support response.
 
 **User Query:** {input}
@@ -863,7 +863,7 @@ report.statistics.rouge_score
 
 Cases where `expected_results` is empty or the agent returned an error are skipped from the average, so a sparse dataset still produces a meaningful score. If a toggle wasn't on for the dataset, that property returns `None`.
 
-These four metrics also appear per-model (as `average_bleu_score`/`average_rouge_score` alongside `average_vector_similarity`/`average_jaccard_similarity`) when a dataset selects multiple comparison models, in each model's row of `report.sovereignty_index.models`.
+These four metrics also appear per-model (as `average_bleu_score`/`average_rouge_score` alongside `average_vector_similarity`/`average_jaccard_similarity`) in each model's row of `report.sovereignty_index.models` when a run compares multiple models. Multi-model comparison comes from the judge scorer's sovereignty models - `client.monitor.judge_scorers.builder(sovereignty_models=[...])` - because self-host ignores `sovereigntyIndex` on the dataset and grading-config routes.
 
 ### CI gate (self-host)
 
@@ -902,7 +902,7 @@ This run + gate flow is the **self-host CI path**. The separate CI-runs API in [
 report = client.evaluations.run(...).execute(my_agent).finalize().analyze(
     mode="auto",                                    # "auto" | "sync" | "batch"
     quality_mode="quality_first",                   # "quality_first" | "balanced"
-    judges=["gpt-5.6-luna", "claude-opus-4-8"],      # 1-3 model ids; omit for the platform default judge
+    judges=["gpt-5.6-luna", "claude-opus-5"],      # 1-3 model ids; omit for the platform default judge
 )
 
 report.summary                 # str | None, overall narrative summary

TRACING.md

@@ -520,6 +520,8 @@ print(pattern.id)
 
 client.monitor.patterns.get(pattern.id)   # -> MonitorPattern
 client.monitor.patterns.list()            # -> list[MonitorPattern]
+client.monitor.patterns.update(pattern.id, enabled=False)  # sparse update, wire camelCase keys -> MonitorPattern
+client.monitor.patterns.delete(pattern.id)                 # historical signals remain as history
 ```
 
 #### `builder()` parameters
@@ -539,9 +541,12 @@ client.monitor.patterns.list()            # -> list[MonitorPattern]
 | `enabled` | `bool` | `True` | Whether the pattern is checked at all |
 | `sample_rate` | `float` | `1.0` | Fraction of matching traces to actually check, `0.0`-`1.0` |
 | `scope_mode` / `agent_ids` | `str` / `list[str]` | `"all"` / `[]` | Restrict this pattern to specific agents instead of the whole workspace |
+| `conditions` | `list[dict]` | `None` | Self-host: the engine's full N-condition model - a list of condition dicts, each with its own detector kind and match settings, passed through verbatim. When set, the engine honors it as the pattern's whole rule set and the flat fields above become display-only metadata |
 
 `publish()` returns a `MonitorPattern` with `.id`, which you pass in `pattern_ids` at trace time.
 
+Note: `match_mode` is write-only on self-host - the engine folds it into the pattern's stored conditions, and the pattern always reads back with `match_mode="any"` regardless of what was sent. The `"all"` semantics still apply when matching.
+
 ### `client.monitor.signals`
 
 Read back the alerts/findings a pattern match (or a built-in detector) produced, without opening the dashboard. Read-only: a signal is the system's output from checking traces against patterns, not something you create directly.

agentx/agentx.py

@@ -4,6 +4,7 @@
 import logging
 
 from agentx.util import get_headers, api_base, normalize_base
+from agentx.exceptions import AgentXError
 from agentx.resources.agent import Agent
 from agentx.resources.workforce import Workforce
 
@@ -136,24 +137,40 @@ def from_env(cls) -> "AgentX":
         return cls(base_url=base_url) if base_url else cls()
 
     def get_agent(self, id: str) -> Agent:
+        """Fetch one hosted-platform agent by id.
+
+        Hosted platform only - the self-host engine does not serve /access/agents;
+        use ``client.monitor.agents.list()`` for self-host agent rows instead.
+        """
         url = f"{self.base_url or api_base()}/access/agents/{id}"
         # Make a GET request to the AgentX API
         response = requests.get(url, headers=get_headers(self.api_key))
         # Check if response was successful
         if response.status_code == 200:
             return Agent(**response.json())
         else:
-            raise Exception(f"Failed to retrieve agent: {response.reason}")
+            raise AgentXError(
+                f"Failed to retrieve agent: {response.reason}. This endpoint is "
+                "hosted-platform only - on self-host use client.monitor.agents.list()."
+            )
 
     def list_agents(self) -> List[Agent]:
+        """List the hosted platform's agents.
+
+        Hosted platform only - the self-host engine does not serve /access/agents;
+        use ``client.monitor.agents.list()`` for self-host agent rows instead.
+        """
         url = f"{self.base_url or api_base()}/access/agents"
         # Make a GET request to the AgentX API
         response = requests.get(url, headers=get_headers(self.api_key))
         # Check if response was successful
         if response.status_code == 200:
             return [Agent(**agent) for agent in response.json()]
         else:
-            raise Exception(f"Failed to list agents: {response.reason}")
+            raise AgentXError(
+                f"Failed to list agents: {response.reason}. This endpoint is "
+                "hosted-platform only - on self-host use client.monitor.agents.list()."
+            )
 
     @staticmethod
     def list_workforces() -> List["Workforce"]:
@@ -214,12 +231,18 @@ def ping(self) -> dict:
         return {"ok": True, "base_url": base}
 
     def get_profile(self):
-        """Get the current user's profile information."""
+        """Get the current user's profile information.
+
+        Hosted platform only - the self-host engine does not serve /access/getProfile;
+        self-host agent/monitoring data lives under ``client.monitor`` (e.g.
+        ``client.monitor.agents.list()``).
+        """
         url = f"{self.base_url or api_base()}/access/getProfile"
         response = requests.get(url, headers=get_headers(self.api_key))
         if response.status_code == 200:
             return response.json()
         else:
-            raise Exception(
-                f"Failed to get profile: {response.status_code} - {response.reason}"
+            raise AgentXError(
+                f"Failed to get profile: {response.status_code} - {response.reason}. "
+                "This endpoint is hosted-platform only - on self-host use client.monitor."
             )

agentx/evaluations/client.py

@@ -35,7 +35,6 @@
 SDK_NAME = "agentx-python"
 
 _RETRYABLE_STATUS = {429, 500, 502, 503, 504}
-_MAX_RETRIES = 3
 _RETRY_BACKOFF = [1.0, 2.0, 4.0]
 
 # The self-host analyze route judges every result before it responds, so the client has to
@@ -182,8 +181,8 @@ def _request(
             if resp.status_code == 422:
                 raise AgentXValidationError(resp.text)
             # Gate on the schedule itself so HTTP-status retries walk the SAME full backoff
-            # schedule connection errors do - the old `attempt < _MAX_RETRIES - 1` gate left
-            # the schedule's last entry unreachable for HTTP retries (ingest_client precedent).
+            # schedule connection errors do - an earlier fixed retry-count gate left the
+            # schedule's last entry unreachable for HTTP retries (ingest_client precedent).
             if (
                 resp.status_code in _RETRYABLE_STATUS
                 and retry
@@ -248,7 +247,9 @@ def delete_dataset(self, dataset_id: str) -> None:
         """Deletes the dataset, its grading config, and both version histories. Past runs are
         kept (their dataset reference degrades to a bare id). The engine refuses (409) when the
         dataset's config is attached to a live scorer."""
-        self._request("DELETE", f"/datasets/{dataset_id}")
+        # retry=False: a lost response + transport retry would turn a successful
+        # delete into a spurious 404.
+        self._request("DELETE", f"/datasets/{dataset_id}", retry=False)
 
     def list_datasets(self) -> List[Dataset]:
         data = self._request("GET", "/datasets", params=self._workspace_params())
@@ -440,6 +441,12 @@ def analyze_run(
         quality_mode: Optional[str] = None,
         judges: Optional[List[str]] = None,
     ) -> Dict[str, Any]:
+        """Start the qualitative AI-analysis job for a run.
+
+        ``mode`` ("auto"/"sync"/"batch") is hosted-only: self-host engines run the analysis
+        synchronously and ignore it - check the response's mode field for what actually ran
+        (mirrors EvaluationRun.analyze's docstring).
+        """
         # Starts the durable analysis job and returns immediately (e.g. {"jobId": ..., "status":
         # "pending"}); poll get_analysis_status() until it reaches a terminal status, then call
         # get_report(). mode/quality_mode/judges mirror the dashboard's AnalyzeEvaluationRequest.
@@ -506,9 +513,10 @@ def get_report(self, run_id: str) -> Report:
         return self._report_from_dashboard(run_id)
 
     def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
-        """Deprecated: on self-host the route's response body has no top-level list, so this
-        always returns ``[]``. Use :meth:`get_submitted_keys` - the same route's
-        ``submittedKeys`` - to find out what a run still needs."""
+        """Deprecated: on self-host the route returns ``missing: []`` deliberately empty -
+        the engine cannot know the client's case list - so this always returns ``[]``.
+        Use :meth:`get_submitted_keys` - the same route's ``submittedKeys`` - to find out
+        what a run still needs."""
         import warnings
 
         warnings.warn(
@@ -517,8 +525,10 @@ def get_missing_results(self, run_id: str) -> List[Dict[str, Any]]:
             DeprecationWarning,
             stacklevel=2,
         )
-        data = self._request("GET", f"/runs/{run_id}/missing-results")
-        return data if isinstance(data, list) else data.get("missing", [])
+        # No request at all: the route returns `missing: []` deliberately empty (the engine
+        # cannot know the client's case list), so the round-trip only ever bought an empty
+        # result.
+        return []
 
     def get_submitted_keys(self, run_id: str) -> List[str]:
         """Idempotency keys this run has already accepted - what execute() uses to resume a

agentx/evaluations/datasets.py

@@ -2,6 +2,7 @@
 
 import csv
 import logging
+import warnings
 from pathlib import Path
 from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union
 
@@ -21,7 +22,8 @@ class DatasetBuilder:
     ``judge_prompt``/``judge_model`` are LLM-as-judge overrides for the dataset's grading
     config. NOTE (self-host): the engine's dataset-create route currently ignores both -
     set them on a judge scorer / the evaluation settings instead. ``sovereignty_models``
-    is accepted on the wire but not acted on by the self-host engine.
+    is dropped by the self-host engine on this route - use
+    ``client.monitor.judge_scorers.builder(sovereignty_models=...)`` which persists it.
     """
 
     def __init__(
@@ -93,8 +95,9 @@ def __init__(
         if rouge_score:
             self._payload["rougeScore"] = {"enabled": True}
         # Sovereignty & Portability - the models to compare on this dataset (use
-        # client.evaluations.list_models() to discover valid ids). Self-host: accepted on
-        # the wire but not acted on by the engine (see class docstring).
+        # client.evaluations.list_models() to discover valid ids). Self-host: dropped by
+        # the engine on this route - use client.monitor.judge_scorers.builder(
+        # sovereignty_models=...) which persists it (see class docstring).
         if sovereignty_models:
             self._payload["sovereigntyIndex"] = {
                 "enabled": True,
@@ -161,9 +164,11 @@ def add_case(
             main["smokeTest"] = {"enabled": True, "count": smoke_test_count}
             if smoke_test_guidance:
                 main["smokeTest"]["guidance"] = smoke_test_guidance
-        if expected_tools:
+        # `is not None`, not truthiness: an explicit empty list is a real assertion (an empty
+        # expectedTrajectory means "this case calls no tools") and must reach the wire.
+        if expected_tools is not None:
             main["expectedTrajectory"] = {"tools": expected_tools, "mode": trajectory_match_mode}
-        if expected_retrieval_context:
+        if expected_retrieval_context is not None:
             main["expectedRetrievalContext"] = expected_retrieval_context
         if splits:
             main["splits"] = splits
@@ -178,6 +183,15 @@ def add_case(
     def publish(self) -> Dataset:
         if not self._payload["questions"]:
             raise ValueError("Dataset must have at least one case before publishing")
+        # Warn at publish time, where the request is known: the engine's dataset-create
+        # route drops sovereigntyIndex, so comparison models set here never persist.
+        sov = self._payload.get("sovereigntyIndex")
+        if isinstance(sov, dict) and sov.get("models"):
+            warnings.warn(
+                "Self-host ignores sovereigntyIndex on datasets/grading configs - use "
+                "judge_scorers.builder(sovereignty_models=...) for model comparison runs.",
+                stacklevel=2,
+            )
         logger.info(
             "Publishing dataset '%s' with %d case(s)",
             self._payload["name"],

agentx/evaluations/evaluation_settings.py

@@ -64,8 +64,9 @@ def __init__(
         if rouge_score:
             self._payload["rougeScore"] = {"enabled": True}
         # Sovereignty & Portability - the models to compare when this config runs
-        # (use client.evaluations.list_models() to discover valid ids). Self-host: accepted
-        # on the wire but not acted on by the engine (same caveat as DatasetBuilder's).
+        # (use client.evaluations.list_models() to discover valid ids). Self-host: dropped
+        # by the engine on this route - use client.monitor.judge_scorers.builder(
+        # sovereignty_models=...) which persists it (same caveat as DatasetBuilder's).
         if sovereignty_models:
             self._payload["sovereigntyIndex"] = {
                 "enabled": True,
@@ -91,6 +92,15 @@ def __init__(
             ]
 
     def publish(self) -> EvaluationSettings:
+        # Warn at publish time, where the request is known: the engine's settings-create
+        # route drops sovereigntyIndex, so comparison models set here never persist.
+        sov = self._payload.get("sovereigntyIndex")
+        if isinstance(sov, dict) and sov.get("models"):
+            warnings.warn(
+                "Self-host ignores sovereigntyIndex on datasets/grading configs - use "
+                "judge_scorers.builder(sovereignty_models=...) for model comparison runs.",
+                stacklevel=2,
+            )
         logger.info("Publishing evaluation settings '%s'", self._payload["name"])
         return self._client.create_evaluation_settings(self._payload)
 

agentx/evaluations/models.py

@@ -179,27 +179,22 @@ class Config:
 # Evaluation subject
 # ---------------------------------------------------------------------------
 
-FrameworkKind = Literal[
-    "raw_python",
-    "openai",
-    "anthropic",
-    "google",
-    "langchain",
-    "llamaindex",
-    "crewai",
-    "autogen",
-    "n8n",
-    "flowise",
-    "other",
-]
-
 RuntimeKind = Literal["local", "ci", "customer_hosted", "low_code"]
 
 
 class EvaluationSubject(BaseModel):
+    """Describes the agent under evaluation.
+
+    ``framework`` is an open string - the engine accepts any label (it also stamps
+    values like ``openai-agents``, ``langgraph``, ``google-genai``, ``litellm`` from
+    the tracing integrations). Common values: ``raw_python``, ``openai``,
+    ``anthropic``, ``google``, ``langchain``, ``llamaindex``, ``crewai``,
+    ``autogen``, ``n8n``, ``flowise``, ``other``.
+    """
+
     kind: Literal["custom_agent", "agentx_agent", "agentx_team"] = "custom_agent"
     display_name: Optional[str] = Field(default=None, alias="displayName")
-    framework: Optional[FrameworkKind] = None
+    framework: Optional[str] = None
     framework_version: Optional[str] = Field(default=None, alias="frameworkVersion")
     runtime: Optional[RuntimeKind] = "local"
     agent_instructions: Optional[str] = Field(default=None, alias="agentInstructions")

agentx/evaluations/reporting.py

@@ -172,7 +172,7 @@ def print_report(report: Report) -> None:
     if report.low_scoring_cases:
         _section("Low-scoring Cases  (rating <= 5)")
         for case in report.low_scoring_cases[:5]:
-            q = (case.get("query") or case.get("questionText", ""))[:80]
+            q = (case.get("query") or case.get("questionText") or "")[:80]
             rating = case.get("rating", "?")
             justification = case.get("justification", "")
             print(f"  {red(f'[{rating}]')} {q}")