add more integration client eval

EVALUATIONS.md

@@ -407,14 +407,14 @@ client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=settings.
 
 #### Configuring the judge
 
-`datasets.builder(...)`, `judge_scorers.builder(...)` and the legacy `settings.builder(...)` all accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike):
+`judge_scorers.builder(...)` and the legacy `settings.builder(...)` accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike). `datasets.builder(...)` accepts the same kwargs for hosted compatibility, but on self-host the engine's dataset-create route ignores both - set them on a judge scorer / the evaluation settings instead (matching the `DatasetBuilder` docstring):
 
 ```python
 scorer = (
     client.monitor.judge_scorers
     .builder(
         name="Strict grading",
-        judge_model="claude-opus-4-8",                    # any id from list_models()
+        judge_model="claude-opus-4-8",                    # any model id your judge key can reach (list_models() on the hosted platform)
         judge_prompt="""You are grading a customer support response.
 
 **User Query:** {input}

README.md

@@ -190,6 +190,13 @@ extra:
 | LlamaIndex            | `pip install "agentx-python[llamaindex]"`    | `AgentXLlamaIndexHandler` |
 | AutoGen               | `pip install "agentx-python[autogen]"`       | `AgentXAutoGenObserver`   |
 
+> **Warning: pick ONE instrumentation layer per LLM call.** Do not combine
+> `AgentXCallbackHandler` (or any framework integration) with a patched provider client
+> (`patch_openai_client`, `patch_anthropic_client`, `patch_genai_client`) on the same code
+> path. A patched call that runs outside an active span emits its own root trace, so every
+> LLM call the framework already traces gets a duplicate trace - and its cost is counted
+> twice.
+
 Two more platforms are covered by **pull importers** rather than in-process hooks, each with its
 own CLI: `agentx-moveworks` (Moveworks Data API sync, no extra needed) and `agentx-databricks`
 (`pip install "agentx-python[databricks]"`, MLflow/Databricks trace sync).

agentx/evaluations/client.py

@@ -221,7 +221,16 @@ def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
         the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
         to filter."""
         params = {"provider": provider} if provider else None
-        data = self._request("GET", "/models", params=params)
+        try:
+            data = self._request("GET", "/models", params=params)
+        except AgentXEvaluationsError as exc:
+            if exc.status_code == 404:
+                raise AgentXEvaluationsError(
+                    "list_models is hosted-only; on self-host pass any model id your judge "
+                    "key can reach, or use client.monitor.* portability models",
+                    status_code=404,
+                ) from exc
+            raise
         items = data if isinstance(data, list) else data.get("models", [])
         return [ModelInfo(**m) for m in items]
 
@@ -546,12 +555,20 @@ def _note_missing_analysis_route(
     ) -> bool:
         """Return True if ``exc`` is the 404 that means "this engine is self-host".
 
-        Only a 404 qualifies. Anything else - auth, validation, a 500, a dead connection -
-        is a real failure on a route that does exist, and must propagate rather than be
-        retried against a different endpoint that would mask it.
+        Only a route-level 404 qualifies. Anything else - auth, validation, a 500, a dead
+        connection - is a real failure on a route that does exist, and must propagate rather
+        than be retried against a different endpoint that would mask it.
+
+        A resource 404 does not qualify either: the engine's SDK router answers these routes
+        with bodies naming the missing resource ("Run not found" / "No analysis found for
+        this run"), so latching on one would permanently reroute every later analysis call
+        to the dashboard router because a caller once passed a wrong run id.
         """
         if exc.status_code != 404:
             return False
+        body = str(exc)
+        if "Run not found" in body or "No analysis found for this run" in body:
+            return False
         if self._analysis_on_dashboard_router is None:
             logger.info(
                 "%s is not served from %s; using the dashboard router at %s "

agentx/evaluations/evaluation_settings.py

@@ -74,8 +74,21 @@ def __init__(
         # Sandboxed JS scorers run per result alongside the judge - each entry is
         # {"name": ..., "enabled": True, "code": "..."} where the code is a JS function body
         # receiving (input, output, expected, toolCalls) and returning {score, reasoning}.
+        # Normalized the same way DatasetBuilder does: id defaulted, name optional (the
+        # engine defaults it), enabled default True - raw pass-through sent entries the
+        # engine's shape validation rejects.
         if code_scorers:
-            self._payload["codeScorers"] = list(code_scorers)
+            import uuid as _uuid
+
+            self._payload["codeScorers"] = [
+                {
+                    "id": scorer.get("id") or _uuid.uuid4().hex[:12],
+                    "name": scorer.get("name"),
+                    "code": scorer["code"],
+                    "enabled": scorer.get("enabled", True),
+                }
+                for scorer in code_scorers
+            ]
 
     def publish(self) -> EvaluationSettings:
         logger.info("Publishing evaluation settings '%s'", self._payload["name"])

agentx/evaluations/models.py

@@ -51,6 +51,10 @@ class TestCase(BaseModel):
     expected_knowledge_base: Optional[List[str]] = Field(default=None, alias="expectedKnowledgeBase")
     expected_delegations: Optional[List[str]] = Field(default=None, alias="expectedDelegations")
     judge_guideline: Optional[str] = Field(default=None, alias="judgeGuideline")
+    # Engine-side trajectory match (e.g. {"tools": ["search"], "mode": "in_order"}) and the
+    # expected retrieval context for RAG grading - carried so import_dataset round-trips them.
+    expected_trajectory: Optional[Dict[str, Any]] = Field(default=None, alias="expectedTrajectory")
+    expected_retrieval_context: Optional[Any] = Field(default=None, alias="expectedRetrievalContext")
     smoke_test: Optional[SmokeTestSettings] = Field(default=None, alias="smokeTest")
     # Named subsets this case belongs to (e.g. ["smoke"], ["full", "regression"]).
     # ``run(dataset_id, split="smoke")`` runs only cases tagged with that split.

agentx/evaluations/reporting.py

@@ -170,7 +170,7 @@ def print_report(report: Report) -> None:
 
     # --- Low-scoring cases ---
     if report.low_scoring_cases:
-        _section("Low-scoring Cases  (rating < 5)")
+        _section("Low-scoring Cases  (rating <= 5)")
         for case in report.low_scoring_cases[:5]:
             q = (case.get("query") or case.get("questionText", ""))[:80]
             rating = case.get("rating", "?")

agentx/evaluations/results.py

@@ -87,6 +87,16 @@ def normalize_result(
     else:
         output = {"text": str(raw)} if raw is not None else {"text": ""}
 
+    if error is None and (
+        output is None
+        or (set(output) <= {"text"} and not str(output.get("text") or "").strip())
+    ):
+        # An empty output with no error would fail the engine's row validation and silently
+        # vanish from the run - store it as an explicit failed row instead.
+        error = ResultError(type="EmptyOutput", message="Agent returned no output")
+        if output is None:
+            output = {"text": ""}
+
     has_timings = (
         latency_ms is not None or input_tokens is not None or output_tokens is not None
     )

agentx/evaluations/runner.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import sys
 import time
 
 import requests
@@ -307,9 +308,23 @@ def bounded() -> "Iterator[EvaluationResult]":
             # generator happens to be garbage-collected.
             if results_iter is not None:
                 results_iter.close()
-
-        if batch:
-            self._flush_batch(batch)
+            # Flush the trailing partial batch HERE, not after the try: a mid-run exception
+            # (agent crash, Ctrl-C) used to discard up to max_batch - 1 already-paid-for
+            # results still waiting in it.
+            if batch:
+                propagating = sys.exc_info()[1]
+                try:
+                    self._flush_batch(batch)
+                except Exception as flush_exc:
+                    if propagating is None:
+                        raise
+                    # An exception is already propagating out of the loop - a flush failure
+                    # here must not mask it.
+                    logger.error(
+                        "Trailing batch flush failed while handling %r: %s",
+                        propagating,
+                        flush_exc,
+                    )
 
         return self
 
@@ -326,6 +341,15 @@ def _flush_batch(self, batch: List[EvaluationResult]) -> None:
                     _say(
                         f"  {green('✓')}  Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}"
                     )
+                    if resp.failed_validation > 0:
+                        # The engine accepts the batch but silently drops rows that fail its
+                        # validation (typically empty output and no error) - say so, or those
+                        # cases just vanish from the report.
+                        _say(
+                            f"  {yellow('!')}  {resp.failed_validation} result"
+                            f"{'s' if resp.failed_validation != 1 else ''} failed validation "
+                            "(empty output and no error) and did not get stored"
+                        )
                     logger.info(
                         "Batch %s: accepted=%d duplicates=%d failed=%d",
                         batch_id[:8],
@@ -415,8 +439,11 @@ def gate(
         ``no_regression=True`` fails it when the average dropped more than ``tolerance``
         (default 0.5, judge scores are noisy) below the dataset's previous completed run.
         At least one check is required. On a multi-judge run, ``scorer`` (an additional
-        scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average
-        instead of the primary's - "fail if Safety is low even when the average looks fine". Prints a CI-log-friendly verdict and returns a
+        judge scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average
+        instead of the primary's - "fail if Safety is low even when the average looks fine".
+        Only judge scorers resolve here: deterministic scorer-group members (pattern/code
+        kinds) have no per-run judge average, so naming one is rejected by the engine.
+        Prints a CI-log-friendly verdict and returns a
         :class:`GateResult` - the caller decides the exit code::
 
             report = client.evaluations.run(...).execute(my_agent).finalize()
@@ -469,6 +496,16 @@ def rated_count(self) -> int:
         """Number of submitted results that have received a rating so far."""
         return self._live_stats.rated_count if self._live_stats else 0
 
+    @property
+    def skipped_count(self) -> int:
+        """Number of submitted results the judge could not score."""
+        return self._live_stats.skipped_count if self._live_stats else 0
+
+    @property
+    def failed_count(self) -> int:
+        """Number of submitted results that carried an error."""
+        return self._live_stats.failed_count if self._live_stats else 0
+
     @property
     def average_rating(self) -> Optional[float]:
         """Live average rating across all results scored so far. Populated as
@@ -639,6 +676,36 @@ def get_analysis_status(self, run_id: str) -> AnalysisStatus:
         script execution)."""
         return self._client.get_analysis_status(run_id)
 
+    # Run-lifecycle calls by id - the standalone forms of what run()/execute()/finalize()/
+    # analyze() drive for you, for scripts operating on a run created elsewhere.
+
+    def init_run(self, dataset_id: str, subject, **kwargs):
+        """Create a run row without executing anything - the standalone form of :meth:`run`.
+        Accepts the same kwargs as ``EvaluationsClient.init_run``."""
+        return self._client.init_run(dataset_id, subject, **kwargs)
+
+    def append_results(self, run_id: str, batch_id: str, results: list):
+        """Submit one batch of results to a run by id (scored synchronously server-side)."""
+        return self._client.append_results(run_id, batch_id, results)
+
+    def finalize_run(self, run_id: str) -> dict:
+        """Mark a run completed by id - the standalone form of
+        ``EvaluationRunContext.finalize()``."""
+        return self._client.finalize_run(run_id)
+
+    def analyze_run(self, run_id: str, **kwargs) -> dict:
+        """Start the LLM analysis of a finalized run by id; poll
+        :meth:`get_analysis_status`, then :meth:`get_report`."""
+        return self._client.analyze_run(run_id, **kwargs)
+
+    def get_report(self, run_id: str):
+        """The analyzed report for a run by id, once analysis has finished."""
+        return self._client.get_report(run_id)
+
+    def get_submitted_keys(self, run_id: str) -> list:
+        """Idempotency keys a run has already accepted - what execute() uses to resume."""
+        return self._client.get_submitted_keys(run_id)
+
     def gate_run(
         self,
         run_id: str,

agentx/integrations/_traced_call.py

@@ -153,7 +153,11 @@ def finish_llm_call(
         )
         return
 
-    span = tracer.trace(name, metadata=metadata, framework=framework, model=model, session_id=session_id)
+    # A patched provider call outside any active span becomes its own root trace - it is a bare
+    # model call, so stamp it "llm" rather than leaving the kind unset.
+    span = tracer.trace(
+        name, metadata=metadata, framework=framework, model=model, session_id=session_id, span_kind="llm"
+    )
     span.__enter__()
     span._start = start_t
     span.input = input_repr

agentx/integrations/autogen.py

@@ -107,7 +107,10 @@ async def run(self, agent_or_team: Any, task: Any = None, **kwargs: Any) -> Any:
             # explicit return/break/continue there would silently swallow any exception
             # propagating from agent_or_team.run() above (see crewai.py's kickoff() for the same
             # hazard spelled out in full).
-            with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
+            # span_kind="agent": the root of a standalone team/agent run is the agent run itself.
+            with self._tracer.trace(
+                self._name, metadata=self._metadata, session_id=self._session_id, span_kind="agent"
+            ) as span:
                 span._start = start_t
                 if error:
                     span.set_error(error)

agentx/integrations/crewai.py

@@ -93,7 +93,10 @@ def kickoff(self, crew: Any, inputs: Optional[Dict[str, Any]] = None) -> Any:
             # `return` here (this whole method body runs inside the try's `finally`) - an
             # explicit return/break/continue in a finally block silently swallows any exception
             # propagating from crew.kickoff() above.
-            with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
+            # span_kind="agent": the root of a standalone crew kickoff is the agent run itself.
+            with self._tracer.trace(
+                self._name, metadata=self._metadata, session_id=self._session_id, span_kind="agent"
+            ) as span:
                 span._start = start
                 if error:
                     span.set_error(error)

agentx/integrations/langchain.py

@@ -595,6 +595,9 @@ def on_chain_end(
                     else self._metadata
                 ),
                 session_id=self._session_id,
+                # The root of a standalone chain/agent invocation is the agent run itself,
+                # not one of its llm/tool/retrieval children.
+                span_kind="agent",
             ) as span:
                 # __enter__ just set _start to "now" - overridden to the chain's real start time,
                 # see llamaindex.py's _send_trace for the identical fix and full rationale.
@@ -655,6 +658,7 @@ def on_chain_error(
                     else self._metadata
                 ),
                 session_id=self._session_id,
+                span_kind="agent",
             ) as span:
                 span._start = state["start"]
                 span.set_error(str(error))

agentx/integrations/llamaindex.py

@@ -359,7 +359,12 @@ def _send_trace(self, state: Dict[str, Any]) -> None:
         # tool_calls loop reads latency_ms for duration and the timestamps for position, so each
         # tool call lands correctly in the tree panel instead of defaulting to offset 0.
         with self._tracer.trace(
-            self._name, metadata=self._metadata, session_id=self._session_id, framework="llamaindex"
+            self._name,
+            metadata=self._metadata,
+            session_id=self._session_id,
+            framework="llamaindex",
+            # The root of a standalone query/agent invocation is the agent run itself.
+            span_kind="agent",
         ) as span:
             # __enter__ just set _start to "now" - overridden to the query's real start time so
             # __exit__'s latency_ms reflects the actual run, not the few microseconds between this