diff --git a/EVALUATIONS.md b/EVALUATIONS.md index 77a3f97..3057819 100644 --- a/EVALUATIONS.md +++ b/EVALUATIONS.md @@ -407,14 +407,14 @@ client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=settings. #### Configuring the judge -`datasets.builder(...)`, `judge_scorers.builder(...)` and the legacy `settings.builder(...)` all accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike): +`judge_scorers.builder(...)` and the legacy `settings.builder(...)` accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike). `datasets.builder(...)` accepts the same kwargs for hosted compatibility, but on self-host the engine's dataset-create route ignores both - set them on a judge scorer / the evaluation settings instead (matching the `DatasetBuilder` docstring): ```python scorer = ( client.monitor.judge_scorers .builder( name="Strict grading", - judge_model="claude-opus-4-8", # any id from list_models() + judge_model="claude-opus-4-8", # any model id your judge key can reach (list_models() on the hosted platform) judge_prompt="""You are grading a customer support response. **User Query:** {input} diff --git a/README.md b/README.md index 64ae21f..47114d0 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,13 @@ extra: | LlamaIndex | `pip install "agentx-python[llamaindex]"` | `AgentXLlamaIndexHandler` | | AutoGen | `pip install "agentx-python[autogen]"` | `AgentXAutoGenObserver` | +> **Warning: pick ONE instrumentation layer per LLM call.** Do not combine +> `AgentXCallbackHandler` (or any framework integration) with a patched provider client +> (`patch_openai_client`, `patch_anthropic_client`, `patch_genai_client`) on the same code +> path. A patched call that runs outside an active span emits its own root trace, so every +> LLM call the framework already traces gets a duplicate trace - and its cost is counted +> twice. + Two more platforms are covered by **pull importers** rather than in-process hooks, each with its own CLI: `agentx-moveworks` (Moveworks Data API sync, no extra needed) and `agentx-databricks` (`pip install "agentx-python[databricks]"`, MLflow/Databricks trace sync). diff --git a/agentx/evaluations/client.py b/agentx/evaluations/client.py index 0dca7ea..e7de74c 100644 --- a/agentx/evaluations/client.py +++ b/agentx/evaluations/client.py @@ -221,7 +221,16 @@ def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]: the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google") to filter.""" params = {"provider": provider} if provider else None - data = self._request("GET", "/models", params=params) + try: + data = self._request("GET", "/models", params=params) + except AgentXEvaluationsError as exc: + if exc.status_code == 404: + raise AgentXEvaluationsError( + "list_models is hosted-only; on self-host pass any model id your judge " + "key can reach, or use client.monitor.* portability models", + status_code=404, + ) from exc + raise items = data if isinstance(data, list) else data.get("models", []) return [ModelInfo(**m) for m in items] @@ -546,12 +555,20 @@ def _note_missing_analysis_route( ) -> bool: """Return True if ``exc`` is the 404 that means "this engine is self-host". - Only a 404 qualifies. Anything else - auth, validation, a 500, a dead connection - - is a real failure on a route that does exist, and must propagate rather than be - retried against a different endpoint that would mask it. + Only a route-level 404 qualifies. Anything else - auth, validation, a 500, a dead + connection - is a real failure on a route that does exist, and must propagate rather + than be retried against a different endpoint that would mask it. + + A resource 404 does not qualify either: the engine's SDK router answers these routes + with bodies naming the missing resource ("Run not found" / "No analysis found for + this run"), so latching on one would permanently reroute every later analysis call + to the dashboard router because a caller once passed a wrong run id. """ if exc.status_code != 404: return False + body = str(exc) + if "Run not found" in body or "No analysis found for this run" in body: + return False if self._analysis_on_dashboard_router is None: logger.info( "%s is not served from %s; using the dashboard router at %s " diff --git a/agentx/evaluations/evaluation_settings.py b/agentx/evaluations/evaluation_settings.py index 67f324a..f6f1dcd 100644 --- a/agentx/evaluations/evaluation_settings.py +++ b/agentx/evaluations/evaluation_settings.py @@ -74,8 +74,21 @@ def __init__( # Sandboxed JS scorers run per result alongside the judge - each entry is # {"name": ..., "enabled": True, "code": "..."} where the code is a JS function body # receiving (input, output, expected, toolCalls) and returning {score, reasoning}. + # Normalized the same way DatasetBuilder does: id defaulted, name optional (the + # engine defaults it), enabled default True - raw pass-through sent entries the + # engine's shape validation rejects. if code_scorers: - self._payload["codeScorers"] = list(code_scorers) + import uuid as _uuid + + self._payload["codeScorers"] = [ + { + "id": scorer.get("id") or _uuid.uuid4().hex[:12], + "name": scorer.get("name"), + "code": scorer["code"], + "enabled": scorer.get("enabled", True), + } + for scorer in code_scorers + ] def publish(self) -> EvaluationSettings: logger.info("Publishing evaluation settings '%s'", self._payload["name"]) diff --git a/agentx/evaluations/models.py b/agentx/evaluations/models.py index e73c21c..8330666 100644 --- a/agentx/evaluations/models.py +++ b/agentx/evaluations/models.py @@ -51,6 +51,10 @@ class TestCase(BaseModel): expected_knowledge_base: Optional[List[str]] = Field(default=None, alias="expectedKnowledgeBase") expected_delegations: Optional[List[str]] = Field(default=None, alias="expectedDelegations") judge_guideline: Optional[str] = Field(default=None, alias="judgeGuideline") + # Engine-side trajectory match (e.g. {"tools": ["search"], "mode": "in_order"}) and the + # expected retrieval context for RAG grading - carried so import_dataset round-trips them. + expected_trajectory: Optional[Dict[str, Any]] = Field(default=None, alias="expectedTrajectory") + expected_retrieval_context: Optional[Any] = Field(default=None, alias="expectedRetrievalContext") smoke_test: Optional[SmokeTestSettings] = Field(default=None, alias="smokeTest") # Named subsets this case belongs to (e.g. ["smoke"], ["full", "regression"]). # ``run(dataset_id, split="smoke")`` runs only cases tagged with that split. diff --git a/agentx/evaluations/reporting.py b/agentx/evaluations/reporting.py index cb49e46..af5b0e7 100644 --- a/agentx/evaluations/reporting.py +++ b/agentx/evaluations/reporting.py @@ -170,7 +170,7 @@ def print_report(report: Report) -> None: # --- Low-scoring cases --- if report.low_scoring_cases: - _section("Low-scoring Cases (rating < 5)") + _section("Low-scoring Cases (rating <= 5)") for case in report.low_scoring_cases[:5]: q = (case.get("query") or case.get("questionText", ""))[:80] rating = case.get("rating", "?") diff --git a/agentx/evaluations/results.py b/agentx/evaluations/results.py index 7296315..dffeb16 100644 --- a/agentx/evaluations/results.py +++ b/agentx/evaluations/results.py @@ -87,6 +87,16 @@ def normalize_result( else: output = {"text": str(raw)} if raw is not None else {"text": ""} + if error is None and ( + output is None + or (set(output) <= {"text"} and not str(output.get("text") or "").strip()) + ): + # An empty output with no error would fail the engine's row validation and silently + # vanish from the run - store it as an explicit failed row instead. + error = ResultError(type="EmptyOutput", message="Agent returned no output") + if output is None: + output = {"text": ""} + has_timings = ( latency_ms is not None or input_tokens is not None or output_tokens is not None ) diff --git a/agentx/evaluations/runner.py b/agentx/evaluations/runner.py index 27bdd9d..232bf2a 100644 --- a/agentx/evaluations/runner.py +++ b/agentx/evaluations/runner.py @@ -2,6 +2,7 @@ import logging import os +import sys import time import requests @@ -307,9 +308,23 @@ def bounded() -> "Iterator[EvaluationResult]": # generator happens to be garbage-collected. if results_iter is not None: results_iter.close() - - if batch: - self._flush_batch(batch) + # Flush the trailing partial batch HERE, not after the try: a mid-run exception + # (agent crash, Ctrl-C) used to discard up to max_batch - 1 already-paid-for + # results still waiting in it. + if batch: + propagating = sys.exc_info()[1] + try: + self._flush_batch(batch) + except Exception as flush_exc: + if propagating is None: + raise + # An exception is already propagating out of the loop - a flush failure + # here must not mask it. + logger.error( + "Trailing batch flush failed while handling %r: %s", + propagating, + flush_exc, + ) return self @@ -326,6 +341,15 @@ def _flush_batch(self, batch: List[EvaluationResult]) -> None: _say( f" {green('✓')} Scored {resp.accepted} result{'s' if resp.accepted != 1 else ''}" ) + if resp.failed_validation > 0: + # The engine accepts the batch but silently drops rows that fail its + # validation (typically empty output and no error) - say so, or those + # cases just vanish from the report. + _say( + f" {yellow('!')} {resp.failed_validation} result" + f"{'s' if resp.failed_validation != 1 else ''} failed validation " + "(empty output and no error) and did not get stored" + ) logger.info( "Batch %s: accepted=%d duplicates=%d failed=%d", batch_id[:8], @@ -415,8 +439,11 @@ def gate( ``no_regression=True`` fails it when the average dropped more than ``tolerance`` (default 0.5, judge scores are noisy) below the dataset's previous completed run. At least one check is required. On a multi-judge run, ``scorer`` (an additional - scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average - instead of the primary's - "fail if Safety is low even when the average looks fine". Prints a CI-log-friendly verdict and returns a + judge scorer's id or name, e.g. ``scorer="Safety"``) gates that scorer's own average + instead of the primary's - "fail if Safety is low even when the average looks fine". + Only judge scorers resolve here: deterministic scorer-group members (pattern/code + kinds) have no per-run judge average, so naming one is rejected by the engine. + Prints a CI-log-friendly verdict and returns a :class:`GateResult` - the caller decides the exit code:: report = client.evaluations.run(...).execute(my_agent).finalize() @@ -469,6 +496,16 @@ def rated_count(self) -> int: """Number of submitted results that have received a rating so far.""" return self._live_stats.rated_count if self._live_stats else 0 + @property + def skipped_count(self) -> int: + """Number of submitted results the judge could not score.""" + return self._live_stats.skipped_count if self._live_stats else 0 + + @property + def failed_count(self) -> int: + """Number of submitted results that carried an error.""" + return self._live_stats.failed_count if self._live_stats else 0 + @property def average_rating(self) -> Optional[float]: """Live average rating across all results scored so far. Populated as @@ -639,6 +676,36 @@ def get_analysis_status(self, run_id: str) -> AnalysisStatus: script execution).""" return self._client.get_analysis_status(run_id) + # Run-lifecycle calls by id - the standalone forms of what run()/execute()/finalize()/ + # analyze() drive for you, for scripts operating on a run created elsewhere. + + def init_run(self, dataset_id: str, subject, **kwargs): + """Create a run row without executing anything - the standalone form of :meth:`run`. + Accepts the same kwargs as ``EvaluationsClient.init_run``.""" + return self._client.init_run(dataset_id, subject, **kwargs) + + def append_results(self, run_id: str, batch_id: str, results: list): + """Submit one batch of results to a run by id (scored synchronously server-side).""" + return self._client.append_results(run_id, batch_id, results) + + def finalize_run(self, run_id: str) -> dict: + """Mark a run completed by id - the standalone form of + ``EvaluationRunContext.finalize()``.""" + return self._client.finalize_run(run_id) + + def analyze_run(self, run_id: str, **kwargs) -> dict: + """Start the LLM analysis of a finalized run by id; poll + :meth:`get_analysis_status`, then :meth:`get_report`.""" + return self._client.analyze_run(run_id, **kwargs) + + def get_report(self, run_id: str): + """The analyzed report for a run by id, once analysis has finished.""" + return self._client.get_report(run_id) + + def get_submitted_keys(self, run_id: str) -> list: + """Idempotency keys a run has already accepted - what execute() uses to resume.""" + return self._client.get_submitted_keys(run_id) + def gate_run( self, run_id: str, diff --git a/agentx/integrations/_traced_call.py b/agentx/integrations/_traced_call.py index 8567f74..bdd5ade 100644 --- a/agentx/integrations/_traced_call.py +++ b/agentx/integrations/_traced_call.py @@ -153,7 +153,11 @@ def finish_llm_call( ) return - span = tracer.trace(name, metadata=metadata, framework=framework, model=model, session_id=session_id) + # A patched provider call outside any active span becomes its own root trace - it is a bare + # model call, so stamp it "llm" rather than leaving the kind unset. + span = tracer.trace( + name, metadata=metadata, framework=framework, model=model, session_id=session_id, span_kind="llm" + ) span.__enter__() span._start = start_t span.input = input_repr diff --git a/agentx/integrations/autogen.py b/agentx/integrations/autogen.py index 3e5dc19..5624aa3 100644 --- a/agentx/integrations/autogen.py +++ b/agentx/integrations/autogen.py @@ -107,7 +107,10 @@ async def run(self, agent_or_team: Any, task: Any = None, **kwargs: Any) -> Any: # explicit return/break/continue there would silently swallow any exception # propagating from agent_or_team.run() above (see crewai.py's kickoff() for the same # hazard spelled out in full). - with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span: + # span_kind="agent": the root of a standalone team/agent run is the agent run itself. + with self._tracer.trace( + self._name, metadata=self._metadata, session_id=self._session_id, span_kind="agent" + ) as span: span._start = start_t if error: span.set_error(error) diff --git a/agentx/integrations/crewai.py b/agentx/integrations/crewai.py index 1c348e3..7c13076 100644 --- a/agentx/integrations/crewai.py +++ b/agentx/integrations/crewai.py @@ -93,7 +93,10 @@ def kickoff(self, crew: Any, inputs: Optional[Dict[str, Any]] = None) -> Any: # `return` here (this whole method body runs inside the try's `finally`) - an # explicit return/break/continue in a finally block silently swallows any exception # propagating from crew.kickoff() above. - with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span: + # span_kind="agent": the root of a standalone crew kickoff is the agent run itself. + with self._tracer.trace( + self._name, metadata=self._metadata, session_id=self._session_id, span_kind="agent" + ) as span: span._start = start if error: span.set_error(error) diff --git a/agentx/integrations/langchain.py b/agentx/integrations/langchain.py index 00bc4f4..268aeaa 100644 --- a/agentx/integrations/langchain.py +++ b/agentx/integrations/langchain.py @@ -595,6 +595,9 @@ def on_chain_end( else self._metadata ), session_id=self._session_id, + # The root of a standalone chain/agent invocation is the agent run itself, + # not one of its llm/tool/retrieval children. + span_kind="agent", ) as span: # __enter__ just set _start to "now" - overridden to the chain's real start time, # see llamaindex.py's _send_trace for the identical fix and full rationale. @@ -655,6 +658,7 @@ def on_chain_error( else self._metadata ), session_id=self._session_id, + span_kind="agent", ) as span: span._start = state["start"] span.set_error(str(error)) diff --git a/agentx/integrations/llamaindex.py b/agentx/integrations/llamaindex.py index 8810290..fae41c8 100644 --- a/agentx/integrations/llamaindex.py +++ b/agentx/integrations/llamaindex.py @@ -359,7 +359,12 @@ def _send_trace(self, state: Dict[str, Any]) -> None: # tool_calls loop reads latency_ms for duration and the timestamps for position, so each # tool call lands correctly in the tree panel instead of defaulting to offset 0. with self._tracer.trace( - self._name, metadata=self._metadata, session_id=self._session_id, framework="llamaindex" + self._name, + metadata=self._metadata, + session_id=self._session_id, + framework="llamaindex", + # The root of a standalone query/agent invocation is the agent run itself. + span_kind="agent", ) as span: # __enter__ just set _start to "now" - overridden to the query's real start time so # __exit__'s latency_ms reflects the actual run, not the few microseconds between this diff --git a/agentx/monitor/judge_scorers.py b/agentx/monitor/judge_scorers.py index f07f447..88ba22f 100644 --- a/agentx/monitor/judge_scorers.py +++ b/agentx/monitor/judge_scorers.py @@ -116,6 +116,7 @@ def builder( judge_prompt: Optional[str] = None, judge_model: Optional[str] = None, tool_context: Optional[str] = None, + requires_expected: Optional[bool] = None, # Offline profile (dataset-run grading) number_of_requests: int = 1, vector_similarity: bool = False, @@ -135,13 +136,17 @@ def builder( alert_threshold: Optional[float] = 5, severity: str = "medium", agent_ids: Optional[List[str]] = None, - idle_seconds: int = 120, + idle_seconds: Optional[int] = None, ) -> "JudgeScorerBuilder": """Snake_case builder with ``.publish()``, the unified successor of ``client.evaluations.settings.builder(...)`` - same offline fields (plus ``thresholds``, ``tool_context``) and, new here, the online profile in the same call. The scorer the builder publishes is one entity: its ``.id`` is what ``client.evaluations.run(..., scorer_id=...)`` takes, and its live profile is what online scoring keys on.""" + if idle_seconds is not None and scope != "session": + # idleSeconds only applies to session scope - with trace scope the engine ignores + # it, so an explicit value here would be silently inert. + raise ValueError("idle_seconds requires scope='session'") judge: Dict[str, Any] = {} for key, value in ( ("acceptanceCriteria", acceptance_criteria), @@ -150,6 +155,7 @@ def builder( ("judgePrompt", judge_prompt), ("judgeModel", judge_model), ("toolContext", tool_context), + ("requiresExpected", requires_expected), ): if value is not None: judge[key] = value @@ -178,7 +184,7 @@ def builder( "scope": scope, "alertThreshold": alert_threshold, "severity": severity, - "idleSeconds": idle_seconds, + "idleSeconds": idle_seconds if idle_seconds is not None else 120, } if agent_ids: online["scopeMode"] = "selected" @@ -329,7 +335,9 @@ def publish_tuning( def ratings(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorRatingPoint]": """Bucketed average-rating-over-time for this scorer's live checks - same typed points - the legacy online_evaluators client returns, so scripts migrate without shape changes.""" + the legacy online_evaluators client returns, so scripts migrate without shape changes. + ``window`` accepts "24h", "7d", or "30d" only (unlike :meth:`calibration`, which also + takes "rubric").""" from agentx.monitor.models import OnlineEvaluatorRatingPoint data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/ratings?window={window}") @@ -337,7 +345,8 @@ def ratings(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorRa def events(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorEvent]": """Individually scored traces behind the ratings series, worst-rated first - typed, same - as the legacy online_evaluators client.""" + as the legacy online_evaluators client. ``window`` accepts "24h", "7d", or "30d" only + (unlike :meth:`calibration`, which also takes "rubric").""" from agentx.monitor.models import OnlineEvaluatorEvent data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/events?window={window}") diff --git a/agentx/monitor/models.py b/agentx/monitor/models.py index 708ac46..1b4996b 100644 --- a/agentx/monitor/models.py +++ b/agentx/monitor/models.py @@ -10,13 +10,14 @@ class MonitorPattern(BaseModel): ``pattern_ids`` entry in ``tracer.trace(..., monitor=True, pattern_ids=[...])``. A "failure" pattern (the default) raises a signal to triage; a "proper" pattern logs a - healthy tally instead. Only one of ``include_terms``/``regex``/``semantic_prompt`` is - meaningful at a time, selected by ``detector_kind``. + healthy tally instead. On self-host the engine stores a pattern as a list of ``conditions`` (each with its own - detector kind and match settings) - the flat ``include_terms``/``regex``/ - ``semantic_prompt`` fields are display-only projections derived from the first condition; - ``conditions`` is the truth. + detector kind and match settings) and ``conditions`` is the only truth: the wire always + carries ``includeTerms``/``excludeTerms`` as ``[]`` and omits ``regex``/``semanticPrompt`` + entirely (they are legacy display fields kept for wire compatibility), so the flat + ``include_terms``/``exclude_terms``/``regex``/``semantic_prompt`` attributes here stay + empty/None - read the match settings from ``conditions``. """ id: str = Field(alias="_id") @@ -41,6 +42,7 @@ class MonitorPattern(BaseModel): agent_ids: List[str] = Field(default_factory=list, alias="agentIds") class Config: + populate_by_name = True extra = "ignore" diff --git a/agentx/tracing/ingest_client.py b/agentx/tracing/ingest_client.py index 56e4b71..ef78c8c 100644 --- a/agentx/tracing/ingest_client.py +++ b/agentx/tracing/ingest_client.py @@ -179,7 +179,9 @@ def send_trace_sync(self, payload: Dict[str, Any]) -> Optional[str]: resp = self._session.post(self._endpoint, json=payload, timeout=10) except requests.RequestException as exc: self._warn_delivery(f"{exc.__class__.__name__}: {exc}") - logger.debug("agentx ingest sync send error: %s", exc) + # WARNING, not debug: the sync caller explicitly asked for a trace_id back, so + # a dropped trace here silently becomes trace_id None downstream. + logger.warning("agentx sync trace send failed (%s) - trace dropped, no trace_id", exc) return None if resp.status_code in (429, 503) and attempt < 2: retry_after = resp.headers.get("Retry-After") @@ -193,7 +195,13 @@ def send_trace_sync(self, payload: Dict[str, Any]) -> Optional[str]: continue if not resp.ok: self._warn_delivery(f"HTTP {resp.status_code}", status=resp.status_code) - logger.debug("agentx ingest sync HTTP %d: %s", resp.status_code, resp.text[:200]) + # WARNING, not debug: the sync caller explicitly asked for a trace_id back, so + # a dropped trace here silently becomes trace_id None downstream. + logger.warning( + "agentx sync trace send failed (HTTP %d: %s) - trace dropped, no trace_id", + resp.status_code, + resp.text[:200], + ) return None try: return resp.json().get("trace_id") @@ -211,11 +219,16 @@ def send_trace_sync_detailed(self, payload: Dict[str, Any]) -> Optional[Dict[str resp = self._session.post(self._endpoint, json=payload, timeout=10) except requests.RequestException as exc: self._warn_delivery(f"{exc.__class__.__name__}: {exc}") - logger.debug("agentx ingest sync send error: %s", exc) + # WARNING, not debug - same reasoning as send_trace_sync. + logger.warning("agentx sync trace send failed (%s) - trace dropped, no trace_id", exc) return None if not resp.ok: self._warn_delivery(f"HTTP {resp.status_code}", status=resp.status_code) - logger.debug("agentx ingest sync HTTP %d: %s", resp.status_code, resp.text[:200]) + logger.warning( + "agentx sync trace send failed (HTTP %d: %s) - trace dropped, no trace_id", + resp.status_code, + resp.text[:200], + ) return None try: body = resp.json() diff --git a/agentx/tracing/tracer.py b/agentx/tracing/tracer.py index 2bd1024..a488d8d 100644 --- a/agentx/tracing/tracer.py +++ b/agentx/tracing/tracer.py @@ -1283,8 +1283,8 @@ def evaluate_trace( (available as ``span.trace_id`` once that `with` block exits). dataset_id: EvaluationSettings ID to score against. question_index: Optional index into the dataset's questions array. - When supplied, that question's ``expectedResults`` - is included in the scoring prompt. + Hosted-only: the self-host engine ignores it and + scores the trace's own input/output as-is. Returns: Dict with keys: ``run_id``, ``trace_id``, ``rating``, diff --git a/tests/test_docs_match_sdk.py b/tests/test_docs_match_sdk.py index 221ae93..d354824 100644 --- a/tests/test_docs_match_sdk.py +++ b/tests/test_docs_match_sdk.py @@ -95,6 +95,21 @@ def test_documented_judge_scorer_methods_exist(): assert not missing, "documented but not on JudgeScorersClient: " + ", ".join(missing) +def test_run_lifecycle_calls_documented_on_client_evaluations_exist(): + """The mintlify API reference shows client.evaluations.(...) for the whole run + lifecycle - these delegate to the private EvaluationsClient, and every one must exist on + the public runner or the documented snippets raise AttributeError.""" + for method in ( + "init_run", + "append_results", + "finalize_run", + "analyze_run", + "get_report", + "get_submitted_keys", + ): + assert hasattr(EvaluationsRunner, method), f"client.evaluations.{method} is documented but missing" + + def test_documented_builder_keywords_are_real_parameters(): keywords = [ (doc, keyword) diff --git a/tests/test_judge_scorers.py b/tests/test_judge_scorers.py index 215bdab..aa4df8e 100644 --- a/tests/test_judge_scorers.py +++ b/tests/test_judge_scorers.py @@ -214,6 +214,39 @@ def fake_request(method, path, **kwargs): assert payload["online"]["agentIds"] == ["support-agent"] +def test_builder_rejects_idle_seconds_without_session_scope(): + """idleSeconds only applies to session scope - the engine silently ignores it with trace + scope, so an explicit idle_seconds without scope="session" is a hard error, not an inert + wire field.""" + client = JudgeScorersClient(api_key="agtx_local_test", base_url="http://localhost:1") + with pytest.raises(ValueError, match="idle_seconds requires scope='session'"): + client.builder("Support quality", live=True, idle_seconds=300) + + +def test_builder_sends_idle_seconds_and_requires_expected(monkeypatch): + """With scope="session", an explicit idle_seconds reaches the wire, and requires_expected + lands in the judge section as requiresExpected.""" + client = JudgeScorersClient(api_key="agtx_local_test", base_url="http://localhost:1") + captured = {} + + def fake_request(method, path, **kwargs): + captured["payload"] = kwargs.get("json") + return {"judgeScorer": {"_id": "s1", "name": "Support quality", "judge": {}, "offline": {}, "online": None}} + + monkeypatch.setattr(client, "_request", fake_request) + client.builder( + "Support quality", + requires_expected=True, + live=True, + scope="session", + idle_seconds=300, + ).publish() + payload = captured["payload"] + assert payload["judge"]["requiresExpected"] is True + assert payload["online"]["scope"] == "session" + assert payload["online"]["idleSeconds"] == 300 + + def test_from_env_honors_selfhost_base_url_conventions(monkeypatch): """from_env silently targeting the hosted default while the shell exports the self-host conventions (AGENTX_SELFHOST_BASE_URL / BASE_URL) produced confusing auth errors - it now diff --git a/tests/test_selfhost_analysis_fallback.py b/tests/test_selfhost_analysis_fallback.py index 5147a05..75f0f92 100644 --- a/tests/test_selfhost_analysis_fallback.py +++ b/tests/test_selfhost_analysis_fallback.py @@ -271,6 +271,27 @@ def test_failures_that_are_not_404_propagate_untouched(status): assert client._analysis_on_dashboard_router is None +@pytest.mark.parametrize("body", [{"error": "Run not found"}, {"error": "No analysis found for this run. POST /runs/:runId/analyze first."}]) +def test_resource_404s_do_not_latch_the_dashboard_fallback(body): + """A 404 whose body names the missing resource comes from a route that EXISTS - the SDK + router answered it. It must propagate as-is and must not permanently reroute every later + analysis call to the dashboard router.""" + client, session = make_client( + { + ("GET", f"{SDK_ROOT}/runs/{RUN}/analyze-status"): FakeResponse(404, body), + # Present, and must not be reached. + ("GET", f"{API_ROOT}/evaluate/analyze/{RUN}/status"): FakeResponse(200, STATUS_BODY), + } + ) + + with pytest.raises(AgentXEvaluationsError) as caught: + client.get_analysis_status(RUN) + + assert caught.value.status_code == 404 + assert not [u for u in session.urls() if "/evaluate/" in u], "masked a resource 404" + assert client._analysis_on_dashboard_router is None + + def test_auth_errors_are_not_mistaken_for_a_missing_route(): client, session = make_client( {("GET", f"{SDK_ROOT}/runs/{RUN}/analyze-status"): FakeResponse(401, {"e": "nope"})}