From 23e028334895bce7909049484089555f1a0874c3 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 14 Aug 2026 06:23:27 -0300 Subject: [PATCH] feat: simplify AgentOps observability Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7a36e0b5-1c09-4953-8e8e-3f8aa2e1f1a9 --- CHANGELOG.md | 31 + pyproject.toml | 3 - src/agentops/agent/checks/catalog.py | 13 - .../agent/checks/posture_rules/__init__.py | 4 - .../aoai_diagnostic_categories.py | 88 -- src/agentops/agent/cockpit.py | 1164 +++++------------ src/agentops/agent/sources/azure_monitor.py | 34 +- src/agentops/cli/app.py | 385 ------ src/agentops/core/agentops_config.py | 119 -- src/agentops/services/agent_identity.py | 11 +- src/agentops/services/dashboard.py | 506 ------- src/agentops/services/telemetry_import.py | 550 -------- src/agentops/templates/workbooks/README.md | 169 --- .../workbooks/foundry-ops.workbook.json | 683 ---------- .../workbooks/queries/agent_behavior.kql | 222 ---- .../queries/capacity_ptu_spillover.kql | 52 - .../workbooks/queries/errors_throttling.kql | 48 - .../workbooks/queries/latency_percentiles.kql | 74 -- .../workbooks/queries/traffic_tokens.kql | 61 - src/agentops/utils/foundry_discovery.py | 85 ++ tests/unit/test_agent_identity_service.py | 10 + tests/unit/test_agent_posture_rules.py | 70 - tests/unit/test_agentops_config.py | 104 -- tests/unit/test_cli_commands.py | 108 +- tests/unit/test_cli_dashboard.py | 156 --- tests/unit/test_cockpit.py | 687 ++++------ tests/unit/test_dashboard.py | 509 ------- tests/unit/test_foundry_discovery.py | 33 + tests/unit/test_telemetry.py | 56 + tests/unit/test_telemetry_import.py | 153 --- 30 files changed, 818 insertions(+), 5370 deletions(-) delete mode 100644 src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py delete mode 100644 src/agentops/services/dashboard.py delete mode 100644 src/agentops/services/telemetry_import.py delete mode 100644 src/agentops/templates/workbooks/README.md delete mode 100644 src/agentops/templates/workbooks/foundry-ops.workbook.json delete mode 100644 src/agentops/templates/workbooks/queries/agent_behavior.kql delete mode 100644 src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql delete mode 100644 src/agentops/templates/workbooks/queries/errors_throttling.kql delete mode 100644 src/agentops/templates/workbooks/queries/latency_percentiles.kql delete mode 100644 src/agentops/templates/workbooks/queries/traffic_tokens.kql delete mode 100644 tests/unit/test_cli_dashboard.py delete mode 100644 tests/unit/test_dashboard.py delete mode 100644 tests/unit/test_telemetry_import.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 00de98c0..eeac4657 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Changed +- **The AgentOps Cockpit now focuses on five release-readiness sections.** + The page renders only the readiness and Doctor ship verdicts, Foundry and + GitHub connections, the 13-item observability checklist, the latest Doctor + findings, and prioritized next actions. Date-window and auto-refresh controls + were removed because the remaining sections describe current configuration + and the latest analysis rather than time-series dashboards. +- **Observability readiness now distinguishes native Foundry tracing from + optional custom spans.** Hosted agents and prompt agents are recognized as + natively instrumented, while repository OpenTelemetry spans are reported as + optional extensions. App Insights linkage, rubric evaluators, alert + definitions, and tracing evidence are resolved independently so unknown state + is no longer presented as a failure. + +### Fixed +- **Project-managed Application Insights connections now work throughout + Doctor and Cockpit.** AgentOps discovers the attached Application Insights + ARM resource from credential-free Foundry connection metadata and queries it + with `LogsQueryClient.query_resource`, avoiding false missing-telemetry + findings when no API-key connection string exists. +- **`agentops agent register` now derives hosted-agent names from Foundry + target URLs.** A command that supplies only `--sponsor` correctly resolves + names such as `helpdeskbot`, matching the fallback promised by `--help`. + +### Removed +- **The experimental AgentOps telemetry import and custom Operations Dashboard + have been removed.** The public telemetry command group, Log Analytics + workbook deployment, bundled workbook/KQL assets, and dashboard-specific + posture rule are no longer shipped while native product observability support + is reviewed. + ## [0.8.8] - 2026-08-14 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 5682a5e6..123a73dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,9 +64,6 @@ where = ["src"] "pipelines/azuredevops/*.yml", "skills/*/SKILL.md", "agent-server/*", - "workbooks/*.json", - "workbooks/*.md", - "workbooks/queries/*.kql", ] "agentops.agent.knowledge" = ["*.csv"] diff --git a/src/agentops/agent/checks/catalog.py b/src/agentops/agent/checks/catalog.py index b25a22ae..05b0baf5 100644 --- a/src/agentops/agent/checks/catalog.py +++ b/src/agentops/agent/checks/catalog.py @@ -775,19 +775,6 @@ def is_llm_judged(self) -> bool: severities=(Severity.WARNING, Severity.CRITICAL), requires=("azure_resources",), ), - CheckSpec( - id="waf.observability.aoai_diagnostic_categories", - category=Category.OPERATIONAL_EXCELLENCE, - title="Azure OpenAI usage telemetry categories are not enabled", - summary=( - "The Azure OpenAI account is not emitting the RequestResponse " - "and AzureOpenAIRequestUsage diagnostic log categories to a Log " - "Analytics workspace, so the Foundry operations dashboard and " - "any token / latency / throttling analysis render empty." - ), - severities=(Severity.WARNING,), - requires=("azure_resources",), - ), # ------------------------------------------------------------------ # Responsible AI # ------------------------------------------------------------------ diff --git a/src/agentops/agent/checks/posture_rules/__init__.py b/src/agentops/agent/checks/posture_rules/__init__.py index 109989a4..fb25927b 100644 --- a/src/agentops/agent/checks/posture_rules/__init__.py +++ b/src/agentops/agent/checks/posture_rules/__init__.py @@ -36,9 +36,6 @@ def _build_registry() -> Dict[str, RuleFn]: from agentops.agent.checks.posture_rules.diagnostics import ( evaluate as diagnostics_rule, ) - from agentops.agent.checks.posture_rules.aoai_diagnostic_categories import ( - evaluate as aoai_diagnostic_categories_rule, - ) from agentops.agent.checks.posture_rules.local_auth import ( evaluate as local_auth_rule, ) @@ -50,7 +47,6 @@ def _build_registry() -> Dict[str, RuleFn]: "waf.security.local_auth_disabled": local_auth_rule, "waf.security.managed_identity": managed_identity_rule, "waf.security.diagnostic_settings": diagnostics_rule, - "waf.observability.aoai_diagnostic_categories": aoai_diagnostic_categories_rule, } diff --git a/src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py b/src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py deleted file mode 100644 index aa97b05c..00000000 --- a/src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py +++ /dev/null @@ -1,88 +0,0 @@ -"""WAF-AI Operational Excellence: Azure OpenAI usage telemetry must flow. - -The Foundry operations dashboard (``agentops telemetry dashboard``) and any -token / latency / throttling analysis depend on two diagnostic log categories -being enabled on the Azure OpenAI (Cognitive Services) account and routed to a -Log Analytics workspace: - -* ``RequestResponse`` - per-request traces (status codes, streaming). -* ``AzureOpenAIRequestUsage`` - prompt / generated token counts. - -When either category is missing the workbook tiles render empty. This rule -fires when the account does not emit both categories and prints the exact -``az monitor diagnostic-settings create`` command to fix it. The check is -read-only; it never changes Azure. -""" - -from __future__ import annotations - -import json -from typing import List - -from agentops.agent.findings import Category, Finding, Severity -from agentops.agent.sources.azure_resources import AzureResourcesPayload - -RULE_ID = "waf.observability.aoai_diagnostic_categories" - -REQUIRED_CATEGORIES = ("RequestResponse", "AzureOpenAIRequestUsage") - - -def _fix_command(account_id: str, workspace_id: str) -> str: - logs = json.dumps([{"category": c, "enabled": True} for c in REQUIRED_CATEGORIES]) - return ( - "az monitor diagnostic-settings create " - "--name agentops-foundry-ops " - f"--resource {account_id} " - f"--workspace {workspace_id} " - f"--logs '{logs}'" - ) - - -def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]: - account = payload.account - if account is None: - return [] - - enabled: set[str] = set() - for setting in payload.diagnostic_settings: - for category in setting.enabled_log_categories: - enabled.add(str(category)) - - missing = [c for c in REQUIRED_CATEGORIES if c not in enabled] - if not missing: - return [] - - account_id = getattr(account, "id", None) or f"<{account.name}-resource-id>" - workspace_id = next( - (s.workspace_id for s in payload.diagnostic_settings if s.workspace_id), - "", - ) - - return [ - Finding( - id=RULE_ID, - severity=Severity.WARNING, - category=Category.OPERATIONAL_EXCELLENCE, - title="Azure OpenAI usage telemetry categories are not enabled", - summary=( - f"Azure OpenAI account `{account.name}` is not emitting the " - f"`{'`, `'.join(missing)}` diagnostic log " - f"{'category' if len(missing) == 1 else 'categories'}. The " - "Foundry operations dashboard and any token, latency, or " - "throttling analysis need both `RequestResponse` and " - "`AzureOpenAIRequestUsage` streamed to a Log Analytics " - "workspace, so those tiles will render empty." - ), - recommendation=( - "Enable the missing categories with:\n" - f"{_fix_command(account_id, workspace_id)}" - ), - source=source_name, - evidence={ - "account": account.name, - "required_categories": list(REQUIRED_CATEGORIES), - "missing_categories": missing, - "enabled_categories": sorted(enabled), - }, - ) - ] diff --git a/src/agentops/agent/cockpit.py b/src/agentops/agent/cockpit.py index cfd07c92..8f5a0b6f 100644 --- a/src/agentops/agent/cockpit.py +++ b/src/agentops/agent/cockpit.py @@ -26,7 +26,7 @@ from urllib.parse import quote from agentops.agent.history import AnalysisRecord, load_analysis_history -from agentops.agent.time_range import TimeRange, parse_time_range, preset_keys +from agentops.agent.time_range import TimeRange from agentops.utils.yaml import load_yaml @@ -70,65 +70,25 @@ def build_cockpit_payload( history: Optional[List[AnalysisRecord]] = None, time_range: Optional[TimeRange] = None, ) -> Dict[str, Any]: - """Reduce raw history + eval runs into a cockpit-ready dict. - - Note: the production section is **not** fetched here. It is rendered - as a placeholder in the initial HTML and filled in asynchronously by - the browser hitting ``/api/production/html``. This keeps the initial - page load fast (local file reads only) even when App Insights is - slow to authenticate or query. - """ - if time_range is None: - time_range = parse_time_range() - all_records = history if history is not None else load_analysis_history(workspace) - records = _filter_records(all_records, time_range) - eval_runs_all = _load_eval_runs(workspace, limit=200) - eval_runs = _filter_eval_runs(eval_runs_all, time_range) + """Reduce local configuration and the latest Doctor run for the cockpit.""" + _ = time_range # Retained for callers; the cockpit no longer filters by date. + records = history if history is not None else load_analysis_history(workspace) telemetry = _telemetry_status() - # Production is deferred to /api/production/html; render a placeholder. - production = {"has_data": False, "deferred": telemetry.get("enabled", False), "cards": []} - - eval_payload = _build_eval_section(eval_runs) - eval_payload["official_eval"] = _official_eval_artifact_status(workspace) watchdog_payload = _build_watchdog_section(records) - deployments_payload = _build_deployments_section(workspace, time_range) - foundry_connection = _build_foundry_connection(workspace, telemetry) - open_in_foundry = _build_open_in_foundry(workspace, telemetry) readiness = _build_readiness_checklist( - workspace, telemetry, deployments_payload, watchdog_payload, + workspace, telemetry, watchdog=watchdog_payload, ) next_actions = _build_next_actions( - workspace, telemetry, watchdog_payload, readiness, eval_payload, + watchdog_payload, readiness, ) return { "workspace": str(workspace.resolve()), - "foundry_project_url": _resolve_foundry_project_url(workspace), - "foundry_compliance_url": _resolve_foundry_compliance_url(workspace), - "foundry_setup_url": _foundry_setup_url(), - "az_tenant_id": _az_tenant_id(), - "time_range": { - "key": time_range.key, - "label": time_range.label, - "start": time_range.start.isoformat(), - "end": time_range.end.isoformat(), - "hours": time_range.hours, - "query": time_range.to_query(), - }, "telemetry": telemetry, - "production": production, - "eval": eval_payload, - "metrics": _build_metrics_cards(eval_runs), "watchdog": watchdog_payload, - "deployments": deployments_payload, - "foundry_connection": foundry_connection, - "open_in_foundry": open_in_foundry, + "connections": _build_connections(workspace), "readiness": readiness, "next_actions": next_actions, - "summary_counts": { - "eval_runs": len(eval_runs), - "analyses": len(records), - }, } @@ -1488,13 +1448,19 @@ def _telemetry_status() -> Dict[str, Any]: } if project: reason: Optional[str] = None + resource_reason: Optional[str] = None try: from agentops.utils.foundry_discovery import ( resolve_appinsights_connection_from_env_with_reason, + resolve_appinsights_resource_id_from_env_with_reason, ) conn, reason = resolve_appinsights_connection_from_env_with_reason() + resource_id, resource_reason = ( + resolve_appinsights_resource_id_from_env_with_reason() + ) except Exception as exc: # noqa: BLE001 conn = None + resource_id = None reason = f"discovery raised {type(exc).__name__}: {exc}" if conn: portal_url = _appinsights_portal_url(conn) @@ -1508,13 +1474,35 @@ def _telemetry_status() -> Dict[str, Any]: "doctor_findings_url": _appinsights_doctor_findings_portal_url(conn), "tone": "ok", } + if resource_id: + return { + "enabled": True, + "source": "foundry_project_connection", + "label": "App Insights", + "detail": ( + "Linked to the Foundry project with Project Managed Identity." + ), + "hint": ( + "Resolved from credential-free Foundry connection metadata; " + "an API key connection string is not required." + ), + "resource_id": resource_id, + "portal_url": _azure_resource_portal_url(resource_id), + "tone": "ok", + } # Surface the actual reason inline so the user does not have to # tail the cockpit server logs to learn why discovery failed. + connection_is_pmi = bool(reason and "ProjectManagedIdentity" in reason) + failure_reason = ( + resource_reason or reason + if connection_is_pmi + else reason or resource_reason + ) reason_html = ( f'
' - f'Why: {_html_escape(reason)}' + f'Why: {_html_escape(failure_reason)}' "
" - if reason + if failure_reason else "" ) return { @@ -1588,6 +1576,11 @@ def _appinsights_portal_url(connection_string: Optional[str]) -> Optional[str]: return _appinsights_logs_url(app_id, query) +def _azure_resource_portal_url(resource_id: str) -> str: + """Build a portal link without requiring App Insights API-key metadata.""" + return f"https://portal.azure.com/#resource{resource_id}/overview" + + def _appinsights_doctor_findings_portal_url(connection_string: Optional[str]) -> Optional[str]: """Build a Logs blade link focused on AgentOps Doctor finding spans.""" if not connection_string: @@ -1668,24 +1661,12 @@ def _url_quote(text: str) -> str: # --------------------------------------------------------------------------- -def _build_foundry_connection( - workspace: Path, - telemetry: Dict[str, Any], -) -> Dict[str, Any]: - """Summarize how this repo connects to Microsoft Foundry. - - Inputs are read-only: env vars, run.yaml/agent.yaml, and the most - recent ``cloud_evaluation.json`` (for the project root). Cockpit - renders this as the first card on the page so users can verify they - are pointed at the right Foundry tenant/project before drilling in. - """ +def _build_connections(workspace: Path) -> Dict[str, Any]: + """Describe the Foundry project and GitHub repository in scope.""" project_env = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") - tenant = _az_tenant_id() project_url = _resolve_foundry_project_url(workspace) project_root = _resolve_foundry_project_root(workspace) - agent_id, agent_source = _resolve_agent_identity(workspace) - if project_env: project_status = "ok" project_label = "Project endpoint configured" @@ -1705,40 +1686,9 @@ def _build_foundry_connection( ) project_copy_value = None - if tenant: - tenant_status = "ok" - tenant_label = "Azure tenant resolved" - tenant_detail = f"{_html_escape(tenant)}" - tenant_hint = "Resolved from `az account show`." - else: - tenant_status = "warn" - tenant_label = "Azure tenant unknown" - tenant_detail = ( - "Run az login so Foundry deep-links open in " - "the correct directory." - ) - tenant_hint = None - - if agent_id: - agent_status = "ok" - agent_label = "Agent configured" - agent_detail = ( - f"{_html_escape(agent_id)}" - ) - agent_hint = f"Resolved from {agent_source}." - else: - agent_status = "muted" - agent_label = "No agent pinned" - agent_detail = ( - "Set agent in agentops.yaml when " - "you want the cockpit to surface a specific Foundry agent." - ) - agent_hint = None - - telemetry_status = telemetry.get("tone", "muted") - telemetry_label = telemetry.get("label", "Telemetry off") - telemetry_detail = telemetry.get("detail", "") - telemetry_hint = telemetry.get("hint") + github = _resolve_github_repository(workspace) + github_name = github.get("name") + github_url = github.get("url") items = [ { @@ -1751,34 +1701,46 @@ def _build_foundry_connection( "copy_value": project_copy_value, }, { - "title": "Azure tenant", - "status": tenant_status, - "label": tenant_label, - "detail": tenant_detail, - "hint": tenant_hint, - }, - { - "title": "Agent", - "status": agent_status, - "label": agent_label, - "detail": agent_detail, - "hint": agent_hint, - "link": (_foundry_deeplinks(workspace).get("agent") if agent_id else None), - "link_label": "Open agent", - }, - { - "title": "Application Insights", - "status": telemetry_status, - "label": telemetry_label, - "detail": telemetry_detail, - "hint": telemetry_hint, - "link": telemetry.get("portal_url"), - "link_label": "Open App Insights", + "title": "GitHub repository", + "status": "ok" if github_url else "warn", + "label": github_name or "GitHub repository missing", + "detail": ( + "Repository remote resolved from local Git configuration." + if github_url + else "Configure an origin remote that points to GitHub." + ), + "link": github_url, + "link_label": "Open in GitHub", + "copy_value": github_url, }, ] + return {"items": items} + + +def _resolve_github_repository(workspace: Path) -> Dict[str, Optional[str]]: + """Resolve a browser URL from the local ``origin`` remote.""" + proc = _run_quick(["git", "remote", "get-url", "origin"], cwd=workspace) + if proc is None or proc.returncode != 0: + return {"name": None, "url": None} + remote = (proc.stdout or "").strip() + if not remote: + return {"name": None, "url": None} + + match = re.match( + r"^(?:https?://|ssh://git@)(?P[^/:]+)[/:](?P.+?)(?:\.git)?$", + remote, + ) + if not match: + match = re.match(r"^git@(?P[^:]+):(?P.+?)(?:\.git)?$", remote) + if not match or "github" not in match.group("host").lower(): + return {"name": None, "url": None} + + path = match.group("path").removesuffix(".git").strip("/") + if path.count("/") != 1: + return {"name": None, "url": None} return { - "items": items, - "has_project": bool(project_env or project_root), + "name": path, + "url": f"https://{match.group('host')}/{path}", } @@ -1791,14 +1753,13 @@ def _build_open_in_foundry( Cockpit surfaces a curated panel of Foundry links so users can drill down without manually navigating the portal. Azure Monitor surfaces - (raw App Insights telemetry and the Foundry operations workbook) are - folded into the Foundry project subgroup so there is a single place to - look, rather than a duplicated one-tile group. + (raw App Insights telemetry) are folded into the Foundry project subgroup + so there is a single place to look, rather than a duplicated one-tile + group. """ deeplinks = _foundry_deeplinks(workspace) portal_url = telemetry.get("portal_url") if isinstance(telemetry, dict) else None project_url = _resolve_foundry_project_url(workspace) - workbook_url = _resolve_workbook_portal_url(workspace) agent_targets: List[Dict[str, Any]] = [ { @@ -1851,17 +1812,6 @@ def _build_open_in_foundry( ), "url": deeplinks.get("operate") or project_url, }, - { - "key": "foundry_ops_dashboard", - "title": "Foundry operations dashboard", - "description": ( - "Azure Monitor workbook with PTU capacity, token traffic, " - "latency percentiles, throttling, and read-only Foundry " - "trace-evaluation behavior. Deploy or open it with " - "`agentops telemetry dashboard`." - ), - "url": workbook_url, - }, { "key": "app_insights", "title": "App Insights", @@ -1891,31 +1841,10 @@ def _build_open_in_foundry( } -def _resolve_workbook_portal_url(workspace: Path) -> Optional[str]: - """Return the Foundry operations workbook portal URL (read-only). - - Mirrors ``agentops telemetry dashboard open``: discovery reads the azd - ``.env`` (a file read, no Azure calls) so the deep link matches. Any - failure falls back to ``None`` so the cockpit never breaks. - """ - try: - from agentops.services import dashboard as dash - - target = dash.discover_target(workspace) - return dash.build_workbook_portal_url( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - name=target.name, - tenant_id=target.tenant_id, - ) - except Exception: # noqa: BLE001 - deep link is best-effort, cockpit stays up - return None - - def _build_readiness_checklist( workspace: Path, telemetry: Dict[str, Any], - deployments: Dict[str, Any], + deployments: Optional[Dict[str, Any]] = None, watchdog: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Read-only checklist of repo-side observability readiness. @@ -1927,6 +1856,7 @@ def _build_readiness_checklist( lives in the Monitor / Evaluations panels users open from the deep-links panel. """ + _ = deployments # Backwards-compatible input; readiness is repo/Doctor based. checks: List[Dict[str, Any]] = [] agentops_config = _read_agentops_config(workspace) trace_manifest = _read_trace_regression_manifest(workspace) @@ -1934,15 +1864,17 @@ def _build_readiness_checklist( trace_lineage: Dict[str, Any] = ( raw_trace_lineage if isinstance(raw_trace_lineage, dict) else {} ) + custom_tracing_path = _detect_custom_tracing(workspace) + foundry_runtime = _detect_foundry_agent_runtime(workspace, agentops_config) - tracing_ok = bool(telemetry.get("enabled")) + tracing_linked = bool(telemetry.get("enabled")) checks.append( { - "title": "Server-side tracing (agent → App Insights)", - "status": "ok" if tracing_ok else "warn", + "title": "App Insights connection", + "status": "ok" if tracing_linked else "warn", "detail": ( - telemetry.get("detail", "") - if tracing_ok + str(telemetry.get("detail") or "Linked to Application Insights.") + if tracing_linked else "How to complete: wire " "APPLICATIONINSIGHTS_CONNECTION_STRING or attach " "App Insights to the Foundry project. " @@ -1952,25 +1884,31 @@ def _build_readiness_checklist( } ) - # Client-side tracing is not auto-detectable from outside the - # caller process — it depends on whether the application itself - # imports the OTel SDK and instruments outbound model / agent - # calls. Surface it as an "info" reminder + docs link so the - # readiness panel stays honest about what AgentOps can verify. - client_side_ok = bool(os.getenv("AGENTOPS_CLIENT_TRACING")) or tracing_ok + agent_tracing_ready = bool(foundry_runtime or custom_tracing_path) checks.append( { - "title": "Client-side tracing (app code instrumented)", - "status": "info" if client_side_ok else "muted", + "title": "Agent tracing instrumentation", + "status": "ok" if agent_tracing_ready else "muted", "detail": ( - "How to complete: instrument the application " - "that calls your agent, not just AgentOps. Add Azure Monitor " - "OpenTelemetry to the app process, configure the same " - "APPLICATIONINSIGHTS_CONNECTION_STRING, and wrap " - "outbound model/agent/tool calls so a user request shows one " - "end-to-end trace across client code, agent execution, and " - "dependencies. Set AGENTOPS_CLIENT_TRACING=1 only " - "after the app is instrumented. " + f"Microsoft Foundry provides native tracing for this " + f"{_html_escape(foundry_runtime)}; no application-side " + "OpenTelemetry setup is required." + + ( + " Additional custom spans were detected in " + f"{_html_escape(custom_tracing_path)}." + if custom_tracing_path + else " Custom spans remain optional." + ) + if foundry_runtime + else ( + "Detected custom OpenTelemetry instrumentation in " + f"{_html_escape(custom_tracing_path)}." + ) + if custom_tracing_path + else "How to complete: instrument the agent " + "runtime with OpenTelemetry. Foundry hosted " + "agents and prompt agents provide this natively; custom or " + "external runtimes must configure their own tracer and exporter. " 'Foundry tracing docs ↗' ), @@ -2011,16 +1949,19 @@ def _build_readiness_checklist( } ) - rubrics = agentops_config.get("rubrics") - rubric_ready = isinstance(rubrics, list) and bool(rubrics) + rubric_ready, rubric_source = _detect_rubric_evaluator( + workspace, + agentops_config, + ) checks.append( { "title": "Optional rubric evaluator gate", "status": "ok" if rubric_ready else "muted", "detail": ( - "Detected rubrics: in agentops.yaml. " - "Keep thresholds bound only to metric names emitted by your " - "Foundry / azd run." + f"Detected a rubric evaluator in " + f"{_html_escape(rubric_source)}. " + "Keep thresholds bound only to metric names emitted by the " + "corresponding Foundry / azd evaluation run." if rubric_ready else "How to complete: optional - add " "rubrics: only after a real Foundry rubric evaluator " @@ -2189,29 +2130,22 @@ def _build_readiness_checklist( } ) - alerts = bool(telemetry.get("portal_url")) and tracing_ok + alerts, alerts_source = _detect_alert_configuration( + workspace, + agentops_config, + ) checks.append( { "title": "Alerts wired", - "status": "info" if alerts else "muted", + "status": "ok" if alerts else "info", "detail": ( - "App Insights is linked. Next, create Azure Monitor alert " - "rules for the agent workload: failures in " - "requests, slow P95 duration, high dependency " - "error rate, and optionally AgentOps CI/Doctor spans such as " - "agentops.eval.* and " - "agentops.agent.finding.*. " - 'Alert docs ↗' + "Detected Azure Monitor alert configuration in " + f"{_html_escape(alerts_source)}." if alerts - else "How to complete: once tracing is wired, " - "create Azure Monitor / App Insights alert rules for the " - "agent workload. Start with requests | where success == false " - "for failures, P95 request duration for latency, and " - "dependencies failures for downstream services. " - "If you want AgentOps signals in alerts too, add rules over " - "agentops.eval.* and " - "agentops.agent.finding.* custom dimensions. " + else "Not verified: Cockpit found no alert definition in repo " + "configuration, and the latest Doctor analysis does not inventory " + "Azure Monitor alert rules. This does not claim that cloud-side " + "alerts are absent. Define alerts as IaC or verify them in Azure Monitor. " 'Alert docs ↗' ), @@ -2296,87 +2230,39 @@ def _continuous_eval_status_from_watchdog( def _build_next_actions( - workspace: Path, - telemetry: Dict[str, Any], watchdog: Dict[str, Any], readiness: Dict[str, Any], - eval_payload: Dict[str, Any], ) -> Dict[str, Any]: - """Surface a short, ordered list of contextual next actions. - - Each action either opens a Cockpit section or links to the relevant - Foundry / Azure runtime view. - """ + """Prioritize Doctor findings, then incomplete readiness checks.""" actions: List[Dict[str, Any]] = [] - - if not telemetry.get("enabled"): - actions.append( - { - "title": "Wire App Insights to Foundry", - "detail": ( - "Tracing is off. Without it Foundry Monitor, " - "Evaluations, and Traces stay empty." - ), - "cta": "Open Foundry connected resources", - "url": _resolve_foundry_project_url(workspace), - } - ) - - readiness_checks = readiness.get("checks", []) - if any(c["status"] == "warn" for c in readiness_checks if c["title"].startswith("CI eval gate")): - actions.append( - { - "title": "Add a CI eval workflow", - "detail": ( - "Generate a PR workflow. Prompt-agent repos use the " - "official Microsoft eval runner when compatible; hosted " - "and fallback cases use agentops eval run." - ), - "cta": "agentops workflow generate", - } - ) - - latest_findings = watchdog.get("latest_findings") or [] - crit_findings = [f for f in latest_findings if (f.get("severity") or "").lower() == "critical"] - if crit_findings: + severity_order = {"critical": 0, "warning": 1, "info": 2} + findings = sorted( + watchdog.get("latest_findings") or [], + key=lambda item: severity_order.get(str(item.get("severity") or "").lower(), 3), + ) + for finding in findings: + recommendation = str(finding.get("recommendation") or finding.get("summary") or "") actions.append( { - "title": f"Fix {len(crit_findings)} critical Doctor finding(s)", - "detail": "Doctor surfaced critical readiness gaps in the repo.", - "cta": "Jump to AgentOps Doctor", + "title": f"Fix Doctor: {finding.get('title') or finding.get('id') or 'finding'}", + "detail": _render_recommendation_body(recommendation), + "cta": "Open Doctor finding", "anchor": "#section-agentops-doctor", } ) - official_eval = _official_eval_artifact_status(workspace) - has_eval_proof = ( - bool(eval_payload.get("has_runs")) - or bool(eval_payload.get("runs")) - or bool(official_eval.get("present")) + readiness_order = {"warn": 0, "muted": 1, "info": 2} + incomplete = sorted( + (check for check in readiness.get("checks", []) if check.get("status") != "ok"), + key=lambda item: readiness_order.get(str(item.get("status") or ""), 3), ) - if not has_eval_proof: + for check in incomplete: actions.append( { - "title": "Run your first evaluation", - "detail": ( - "No eval gate evidence yet. Run agentops eval run " - "locally, or run the generated official-eval workflow for a " - "compatible Foundry prompt agent." - ), - "cta": "agentops eval run", - } - ) - - evidence = _release_evidence_status(workspace) - if has_eval_proof and evidence.get("status") in {"missing", "unreadable"}: - actions.append( - { - "title": "Generate release evidence", - "detail": ( - "Package the latest eval gate, Doctor findings, CI/CD " - "status, and Foundry links into the release-review artifact." - ), - "cta": "agentops doctor --evidence-pack", + "title": f"Complete readiness: {check.get('title') or 'configuration'}", + "detail": check.get("detail", ""), + "cta": "Open readiness item", + "anchor": "#section-readiness", } ) @@ -2385,8 +2271,7 @@ def _build_next_actions( { "title": "All caught up", "detail": ( - "No outstanding readiness gaps detected in the repo. " - "Use the Foundry deep-links above to monitor runtime." + "All readiness items are complete and Doctor has no findings." ), "cta": None, } @@ -2610,6 +2495,142 @@ def _read_agentops_config(workspace: Path) -> Dict[str, Any]: return payload if isinstance(payload, dict) else {} +def _detect_foundry_agent_runtime( + workspace: Path, + agentops_config: Dict[str, Any], +) -> Optional[str]: + """Identify Foundry runtimes that provide native agent tracing.""" + path = workspace / "azure.yaml" + if path.is_file(): + try: + payload = load_yaml(path) + except Exception: + payload = None + if isinstance(payload, dict): + services = payload.get("services") + if isinstance(services, dict) and any( + isinstance(service, dict) + and service.get("host") == "azure.ai.agent" + and service.get("kind") == "hosted" + for service in services.values() + ): + return "hosted agent runtime" + + raw_agent = agentops_config.get("agent") + if isinstance(raw_agent, str) and raw_agent.strip(): + try: + from agentops.core.agentops_config import classify_agent + + target = classify_agent( + raw_agent, + protocol=agentops_config.get("protocol"), + ) + except (TypeError, ValueError): + target = None + if target is not None: + if target.kind == "foundry_hosted": + return "hosted agent runtime" + if target.kind == "foundry_prompt": + return "prompt agent runtime" + + for eval_path in (workspace / "src").glob("**/eval.y*ml"): + try: + eval_payload = load_yaml(eval_path) + except Exception: + continue + if not isinstance(eval_payload, dict): + continue + agent = eval_payload.get("agent") + kind = agent.get("kind") if isinstance(agent, dict) else None + if kind == "hosted": + return "hosted agent runtime" + if kind in {"prompt", "prompt-agent"}: + return "prompt agent runtime" + return None + + +def _detect_custom_tracing(workspace: Path) -> Optional[str]: + """Find optional repository code that emits custom OpenTelemetry spans.""" + candidates = list((workspace / "src").glob("**/*.py")) + candidates.extend(workspace.glob("*.py")) + for path in candidates: + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + has_otel = "opentelemetry" in text or "configure_azure_monitor" in text + emits_spans = any( + marker in text + for marker in ( + "start_as_current_span", + "start_span(", + "get_tracer(", + "configure_azure_monitor(", + ) + ) + if has_otel and emits_spans: + return path.relative_to(workspace).as_posix() + return None + + +def _detect_rubric_evaluator( + workspace: Path, + agentops_config: Dict[str, Any], +) -> Tuple[bool, str]: + """Resolve rubric evidence from AgentOps or azd AI Agent eval config.""" + rubrics = agentops_config.get("rubrics") + if isinstance(rubrics, list) and rubrics: + return True, "agentops.yaml" + + for path in (workspace / "src").glob("**/eval.y*ml"): + try: + payload = load_yaml(path) + except Exception: + continue + if not isinstance(payload, dict): + continue + evaluators = payload.get("evaluators") + if isinstance(evaluators, list) and any( + isinstance(evaluator, dict) + and bool(evaluator.get("local_uri") or evaluator.get("id")) + for evaluator in evaluators + ): + return True, path.relative_to(workspace).as_posix() + return False, "" + + +def _detect_alert_configuration( + workspace: Path, + agentops_config: Dict[str, Any], +) -> Tuple[bool, str]: + """Find explicit alert definitions without claiming cloud state.""" + observability = agentops_config.get("observability") + if isinstance(observability, dict) and observability.get("alerts"): + return True, "agentops.yaml" + + markers = ( + "microsoft.insights/metricalerts", + "microsoft.insights/scheduledqueryrules", + "azurerm_monitor_metric_alert", + "azurerm_monitor_scheduled_query_rules_alert", + ) + candidates: List[Path] = [] + for root in (workspace / "infra", workspace / "deploy"): + if root.is_dir(): + for pattern in ("**/*.bicep", "**/*.tf", "**/*.json", "**/*.y*ml"): + candidates.extend(root.glob(pattern)) + for pattern in ("*.bicep", "*.tf"): + candidates.extend(workspace.glob(pattern)) + for path in candidates: + try: + text = path.read_text(encoding="utf-8", errors="ignore").lower() + except OSError: + continue + if any(marker in text for marker in markers): + return True, path.relative_to(workspace).as_posix() + return False, "" + + def _read_trace_regression_manifest(workspace: Path) -> Dict[str, Any]: return _read_json_object(workspace / ".agentops" / "data" / "trace-regression-manifest.json") @@ -3468,10 +3489,8 @@ def _collapsible_section( def _render_status_cards_section( readiness: Dict[str, Any], watchdog: Dict[str, Any], - eval_payload: Dict[str, Any], ) -> str: - """Render the consolidated top status: three clickable cards that - answer "can I ship?" at a glance (Readiness, Doctor, Eval gate). + """Render the two go/no-go verdicts that answer "Can I ship?". Each card is an anchor to the matching detail section below; the template's hash-open script expands that (collapsed) section when the @@ -3503,9 +3522,9 @@ def _card( readiness_tone = "warn" readiness_card = _card( title="Readiness", - value=readiness_label, + value="GO" if readiness_tone == "ok" else "NO-GO", tone=readiness_tone, - sub="Observability checks green", + sub=readiness_label, anchor="#section-readiness", ) @@ -3521,9 +3540,9 @@ def _headline_value(key: str) -> int: return 0 if not watchdog.get("has_history"): - doctor_tone = "muted" - doctor_value = "No runs" - doctor_sub = "Run agentops doctor" + doctor_tone = "warn" + doctor_value = "NO-GO" + doctor_sub = "No Doctor run" else: findings_total = _headline_value("findings_total") critical = _headline_value("critical") @@ -3533,7 +3552,7 @@ def _headline_value(key: str) -> int: doctor_tone = "warn" else: doctor_tone = "ok" - doctor_value = f"{findings_total} finding(s)" + doctor_value = "GO" if findings_total == 0 else "NO-GO" doctor_sub = f"{critical} critical" doctor_card = _card( title="Doctor", @@ -3543,28 +3562,7 @@ def _headline_value(key: str) -> int: anchor="#section-agentops-doctor", ) - if not eval_payload.get("has_runs"): - eval_tone = "muted" - eval_value = "No runs" - eval_sub = "Run agentops eval run" - else: - passed = bool(eval_payload.get("latest_passed")) - eval_tone = "ok" if passed else "crit" - eval_value = "Pass" if passed else "Fail" - pass_rate = eval_payload.get("pass_rate") - if isinstance(pass_rate, (int, float)): - eval_sub = f"{int(pass_rate * 100)}% pass rate" - else: - eval_sub = "Latest run" - eval_card = _card( - title="Eval gate", - value=eval_value, - tone=eval_tone, - sub=eval_sub, - anchor="#section-eval-gates", - ) - - cards = readiness_card + doctor_card + eval_card + cards = readiness_card + doctor_card return ( '
' '
Can I ship? ' @@ -3636,7 +3634,7 @@ def _render_foundry_connection_section(connection: Dict[str, Any]) -> str: ) body = f'
{"".join(items_html)}
' return _collapsible_section( - "Foundry connection", body, section_id="section-foundry-connection" + "Connections", body, section_id="section-connections" ) @@ -3914,121 +3912,6 @@ def render_cockpit_html(payload: Dict[str, Any]) -> str: :func:`build_cockpit_payload`. Returns a complete HTML document. """ telemetry = payload["telemetry"] - # Show the telemetry card only when telemetry is OFF - it then acts as - # the "why is the production section empty" hint. When telemetry is - # active, the dedicated Production signal section communicates the - # connection state already. - show_telemetry_card = not telemetry.get("enabled", False) - telemetry_card = _render_telemetry_card(telemetry) if show_telemetry_card else "" - eval_runs_url = ( - telemetry.get("eval_runs_url") if isinstance(telemetry, dict) else None - ) - eval_link = "" - if eval_runs_url: - eval_link = ( - f' ' - f'View CI evals in App Insights →' - ) - - eval_body = "" - eval_subtitle = "Eval gate summary" - eval_caption = ( - '
' - 'AgentOps gate history from local artifacts and CI runs. For ' - 'Foundry cloud runs, use this as the quick pass/fail ' - 'triage view and open Foundry Evaluations for full analysis.' - '
' - ) - if payload["eval"]["has_runs"]: - cards_html = "".join(_render_card(c) for c in payload["eval"]["cards"]) - exec_tag = _render_exec_section_tag( - payload["eval"].get("latest_execution"), - ) - eval_body = f'{eval_caption}
{cards_html}{telemetry_card}
' - eval_subtitle = f"Eval gate summary{exec_tag}" - else: - official_eval = payload["eval"].get("official_eval") or {} - if official_eval.get("present"): - status = _html_escape(official_eval.get("status") or "metadata-only") - empty_text = ( - "No AgentOps-normalized eval runs yet under " - ".agentops/results/. Official Microsoft Foundry " - "AI Agent Evaluation evidence exists under " - f".agentops/official-eval/ with status {status}. " - "Run agentops doctor --evidence-pack to package it " - "for release review." - ) - else: - empty_text = ( - "No eval runs yet under .agentops/results/. " - "Run agentops eval run to populate this section." - ) - eval_body = ( - eval_caption + - '
' - f"{empty_text}" - "
" - + (f'
{telemetry_card}
' if telemetry_card else "") - ) - eval_subtitle = "Eval gate summary" - - deployments = payload.get("deployments") or {} - if deployments.get("has_data") and deployments.get("cards"): - deploy_cards = "".join(_render_card(c) for c in deployments["cards"]) - deployments_body = f'
{deploy_cards}
' - else: - hint = deployments.get("hint") or ( - "Install the GitHub CLI and run gh auth login " - "to surface workflow runs here." - ) - deployments_body = f'
{hint}
' - deployments_section = _collapsible_section( - "CI/CD Pipelines", deployments_body, section_id="section-cicd", - open_by_default=False, - ) - - metrics_subtitle = "Quality gate summary" - metrics_body = "" - if payload["metrics"]: - metrics_html = "".join(_render_card(c) for c in payload["metrics"]) - exec_tag = _render_exec_section_tag( - payload["eval"].get("latest_execution") if payload["eval"].get("has_runs") else None, - ) - metrics_caption = ( - '
' - 'Quality gate trends computed from AgentOps result artifacts. ' - 'Keep this as a compact threshold/regression summary; detailed ' - 'cloud-evaluation drilldown belongs in Foundry Evaluations.' - '
' - ) - metrics_body = f'{metrics_caption}
{metrics_html}
' - metrics_subtitle = f"Quality gate summary{exec_tag}" - - # Merge the eval gate + quality gate into a single "Eval gates" section - # with two subgroups. They share a source (AgentOps result artifacts) - # and CTA (View CI evals in App Insights), so folding them removes a - # redundant top-level section and keeps the gate story in one place. - eval_gate_subgroup = ( - '' - ) - quality_gate_subgroup = "" - if metrics_body: - quality_gate_subgroup = ( - '' - ) - eval_gates_section = _collapsible_section( - f"Eval gates{eval_link}", - eval_gate_subgroup + quality_gate_subgroup, - section_id="section-eval-gates", - open_by_default=False, - ) watchdog = payload["watchdog"] watchdog_title = "AgentOps Doctor" @@ -4063,114 +3946,9 @@ def render_cockpit_html(payload: Dict[str, Any]) -> str: open_by_default=False, ) - production = payload.get("production") or {} - production_section = "" - # The App Insights KQL portal URL is already surfaced by the launchpad - # "App Insights" tile and the Doctor / Eval gate "View in App Insights" - # links, so we no longer repeat it here. Production signal keeps a - # single primary CTA: the full Foundry Monitor view. - _portal_url_unused = telemetry.get("portal_url") if isinstance(telemetry, dict) else None - - # Cockpit surfaces a 2-card teaser (error rate + P95); this link is - # the primary call-to-action for the full Foundry Monitor view. - foundry_monitor_url = None - open_panel = payload.get("open_in_foundry") or {} - for target in open_panel.get("targets", []): - if target.get("key") == "monitor": - foundry_monitor_url = target.get("url") - break - foundry_monitor_link = "" - if foundry_monitor_url: - foundry_monitor_link = ( - f' ' - 'Full view in Foundry Monitor →' - ) - - prod_title = ( - 'Production signal' - ' live · App Insights' - f'{foundry_monitor_link}' - ) - if production.get("has_data") and production.get("cards"): - # Server-side render (rare - happens when /api/production/html is - # invoked directly without a deferred placeholder). - prod_html = "".join(_render_card(c, hero=True) for c in production["cards"]) - prod_caption = ( - '
' - 'Fast health snapshot from App Insights. Use ' - 'Foundry Monitor for the full production view ' - '(invocations, tokens, per-model breakdown). Raw App Insights ' - 'KQL is available from the launchpad "App Insights" tile.' - '
' - ) - prod_body = f'{prod_caption}
{prod_html}
' - production_section = _collapsible_section( - prod_title, prod_body, open_by_default=False, - ) - elif production.get("deferred"): - # Telemetry is wired up; the cards will arrive async from - # /api/production/html so the page can render immediately. - # Render 2 skeleton cards matching the teaser layout - # (Error rate / P95 latency). Invocations and tokens - # intentionally live in Foundry Monitor only. - skeleton_labels = ("Error rate", "P95 latency") - skeleton_cards = "".join( - ( - '
' - f'
{label}
' - '
' - '
' - '
' - '
' - ) - for label in skeleton_labels - ) - prod_caption = ( - '
' - 'Fast health snapshot from App Insights. Use ' - 'Foundry Monitor for the full production view ' - '(invocations, tokens, per-model breakdown). Raw App Insights ' - 'KQL is available from the launchpad "App Insights" tile.' - '
' - ) - prod_body = ( - f'{prod_caption}' - f'
{skeleton_cards}
' - ) - production_section = _collapsible_section( - prod_title, prod_body, open_by_default=False, - ) - - counts = payload["summary_counts"] workspace_display = _shorten_workspace(payload["workspace"]) - range_info = payload.get("time_range") or {} - range_bar = _render_range_bar(range_info) - - foundry_uri = _foundry_logo_data_uri() - foundry_url = payload.get("foundry_project_url") or "https://ai.azure.com" - if foundry_uri: - powered_by_html = ( - f'' - f'Foundry' - 'Your Foundry project ↗' - '' - ) - else: - powered_by_html = "" - - # Banner removed; the primer click is now folded into the project - # action button above, which carries the ?tid= hint. - setup_banner_html = "" - - foundry_connection_section = _render_foundry_connection_section( - payload.get("foundry_connection") or {"items": []} - ) - open_in_foundry_section = _render_open_in_foundry_section( - payload.get("open_in_foundry") or {"targets": []} + connections_section = _render_foundry_connection_section( + payload.get("connections") or {"items": []} ) readiness_section = _render_readiness_section( payload.get("readiness") or {"checks": [], "label": "0/0 ready"} @@ -4181,98 +3959,20 @@ def render_cockpit_html(payload: Dict[str, Any]) -> str: status_cards_section = _render_status_cards_section( payload.get("readiness") or {"checks": [], "label": "0/0 ready"}, payload.get("watchdog") or {}, - payload.get("eval") or {}, ) return _COCKPIT_TEMPLATE.format( - foundry_connection_section=foundry_connection_section, - open_in_foundry_section=open_in_foundry_section, status_cards_section=status_cards_section, + connections_section=connections_section, readiness_section=readiness_section, - next_actions_section=next_actions_section, - eval_gates_section=eval_gates_section, - deployments_section=deployments_section, - production_section=production_section, watchdog_section=watchdog_section, - eval_runs=counts["eval_runs"], - analyses=counts["analyses"], + next_actions_section=next_actions_section, workspace_display=workspace_display, workspace=payload["workspace"], icon_uri=_icon_data_uri(), - powered_by=powered_by_html, - setup_banner=setup_banner_html, - range_bar=range_bar, - range_label=_html_escape(range_info.get("label", "")), - ) - - -def _render_range_bar(range_info: Dict[str, Any]) -> str: - """Render the 1D / 7D / 30D / Custom selector.""" - active_key = range_info.get("key", "7d") - pills: List[str] = [] - labels = {"1d": "1D", "7d": "7D", "30d": "30D"} - for key in preset_keys(): - cls = "range-pill active" if key == active_key else "range-pill" - pills.append(f'{labels[key]}') - custom_cls = "range-pill active" if active_key == "custom" else "range-pill" - pills.append( - f'Custom' - ) - - today = _today_iso() - week_ago = _days_ago_iso(7) - custom_from = range_info.get("start", "")[:10] if active_key == "custom" else week_ago - custom_to = range_info.get("end", "")[:10] if active_key == "custom" else today - form_class = "range-custom-form open" if active_key == "custom" else "range-custom-form" - - custom_form = ( - f'
' - f'' - f'' - f'' - f'' - f'
' - ) - - refresh_control = ( - '
' - '' - 'Refresh' - '' - '
' - ) - - return ( - '
' - + '
' + "".join(pills) + '
' - + custom_form - + refresh_control - + '
' ) -def _today_iso() -> str: - from datetime import datetime, timezone - return datetime.now(timezone.utc).strftime("%Y-%m-%d") - - -def _days_ago_iso(days: int) -> str: - from datetime import datetime, timedelta, timezone - return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d") - - def _shorten_workspace(path: str) -> str: """Show only the current folder name for compact heading display. @@ -4736,91 +4436,6 @@ def _shorten_workspace(path: str) -> str: 0%, 100% {{ opacity: 1; }} 50% {{ opacity: 0.6; }} }} - .range-bar {{ - display: flex; align-items: center; flex-wrap: wrap; gap: 12px; - margin-bottom: 8px; - }} - .range-pills {{ display: flex; gap: 4px; }} - .range-pill {{ - padding: 6px 14px; border-radius: 999px; - color: var(--text-dim); text-decoration: none; - font-size: 12px; font-weight: 600; letter-spacing: 0.02em; - background: rgba(255, 255, 255, 0.03); - border: 1px solid var(--border); - transition: background 0.15s ease, color 0.15s ease; - }} - .range-pill:hover {{ - background: rgba(255, 255, 255, 0.06); color: var(--text); - }} - .range-pill.active {{ - background: rgba(56, 189, 248, 0.14); color: var(--info); - border-color: rgba(56, 189, 248, 0.35); - }} - .range-custom-form {{ - display: none; gap: 10px; align-items: center; - background: var(--card); padding: 10px 14px; - border-radius: 999px; border: 1px solid var(--border); - font-size: 12px; color: var(--text-dim); - }} - .range-custom-form.open {{ display: flex; }} - .range-custom-form input[type="date"] {{ - background: rgba(255, 255, 255, 0.04); - border: 1px solid var(--border); color: var(--text); - padding: 4px 8px; border-radius: 8px; font-size: 12px; - color-scheme: dark; - }} - .range-custom-form button {{ - background: var(--info); color: #08090b; border: 0; - padding: 5px 12px; border-radius: 999px; - font-size: 12px; font-weight: 700; cursor: pointer; - }} - .range-current {{ - color: var(--text-faint); font-size: 11px; font-weight: 500; - margin-left: auto; - font-family: "SF Mono", "Cascadia Code", Consolas, monospace; - }} - .refresh-control {{ - margin-left: auto; - display: inline-flex; align-items: center; gap: 8px; - padding: 5px 10px 5px 12px; border-radius: 999px; - background: rgba(255, 255, 255, 0.03); - border: 1px solid var(--border); - color: var(--text-dim); - font-size: 12px; font-weight: 600; letter-spacing: 0.02em; - transition: background 0.15s ease, color 0.15s ease, - border-color 0.15s ease; - }} - .refresh-control:hover {{ - background: rgba(255, 255, 255, 0.06); color: var(--text); - border-color: rgba(56, 189, 248, 0.35); - }} - .refresh-control .refresh-icon {{ - color: var(--info); opacity: 0.9; - }} - .refresh-control.spinning .refresh-icon {{ - animation: refresh-spin 0.8s linear; - }} - .refresh-label {{ - color: var(--text-faint); text-transform: uppercase; - font-size: 10px; letter-spacing: 0.06em; - }} - #refreshSelect {{ - appearance: none; -webkit-appearance: none; - background: transparent; color: var(--text); - border: 0; outline: none; - font: inherit; font-weight: 600; - padding: 0 18px 0 2px; cursor: pointer; - background-image: url("data:image/svg+xml;utf8,"); - background-repeat: no-repeat; - background-position: right 2px center; - }} - #refreshSelect option {{ - background: var(--card); color: var(--text); - }} - @keyframes refresh-spin {{ - from {{ transform: rotate(0deg); }} - to {{ transform: rotate(360deg); }} - }} .grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 14px; @@ -5207,84 +4822,17 @@ def _shorten_workspace(path: str) -> str:
{workspace_display}
-
-
-
{eval_runs} eval(s)
-
{analyses} analysis run(s)
-
- {powered_by} -
-{setup_banner} - -{range_bar} -
window: {range_label}
- -{foundry_connection_section} {status_cards_section} -{next_actions_section} +{connections_section} {readiness_section} {watchdog_section} -{eval_gates_section} -{production_section} -{deployments_section} -{open_in_foundry_section} +{next_actions_section} -
Auto-refresh: every 5 min · agentops cockpit
+
agentops cockpit
@@ -5425,21 +4913,12 @@ def create_app(workspace: Path): @app.get("/", response_class=HTMLResponse) def _index( - range_: Optional[str] = Query(None, alias="range"), - from_: Optional[str] = Query(None, alias="from"), - to: Optional[str] = Query(None), partial: Optional[str] = Query(None, alias="_partial"), ) -> HTMLResponse: - # The full render does file IO, history aggregation, and a - # `gh run list` subprocess for CI/CD, which together can take - # several seconds. To avoid a "black page" while the browser - # waits, ``/`` returns a tiny branded shell with an animated - # loader, and the shell fetches ``/?_partial=1`` to hydrate - # the real cockpit once the heavy work completes. + # Return a tiny shell immediately, then hydrate the local cockpit. if not partial: return HTMLResponse(_render_loading_shell()) - time_range = parse_time_range(range_, from_, to) - payload = build_cockpit_payload(workspace, time_range=time_range) + payload = build_cockpit_payload(workspace) return HTMLResponse(render_cockpit_html(payload)) @app.get("/favicon.ico") @@ -5468,25 +4947,6 @@ def _api_run_report(run_id: str) -> HTMLResponse: def _api_telemetry() -> JSONResponse: return JSONResponse(_telemetry_status()) - @app.get("/api/production") - def _api_production( - range_: Optional[str] = Query(None, alias="range"), - from_: Optional[str] = Query(None, alias="from"), - to: Optional[str] = Query(None), - ) -> JSONResponse: - time_range = parse_time_range(range_, from_, to) - return JSONResponse(_build_production_section(_telemetry_status(), time_range=time_range)) - - @app.get("/api/production/html", response_class=HTMLResponse) - def _api_production_html( - range_: Optional[str] = Query(None, alias="range"), - from_: Optional[str] = Query(None, alias="from"), - to: Optional[str] = Query(None), - ) -> HTMLResponse: - time_range = parse_time_range(range_, from_, to) - production = _build_production_section(_telemetry_status(), time_range=time_range) - return HTMLResponse(render_production_grid_html(production)) - @app.get("/healthz") def _healthz() -> Dict[str, str]: return {"status": "ok"} diff --git a/src/agentops/agent/sources/azure_monitor.py b/src/agentops/agent/sources/azure_monitor.py index 42185e30..202e5761 100644 --- a/src/agentops/agent/sources/azure_monitor.py +++ b/src/agentops/agent/sources/azure_monitor.py @@ -105,12 +105,29 @@ def collect_azure_monitor( ) -> AzureMonitorPayload: """Run KQL queries against Application Insights for the lookback window.""" diagnostics: Dict[str, Any] = {"enabled": config.enabled} + app_insights_resource_id = config.app_insights_resource_id if not config.enabled: diagnostics["status"] = "disabled" return AzureMonitorPayload(diagnostics=diagnostics) - if not config.app_insights_resource_id and not config.log_analytics_workspace_id: + if not app_insights_resource_id and not config.log_analytics_workspace_id: + resource_reason: Optional[str] = None + try: + from agentops.utils.foundry_discovery import ( + resolve_appinsights_resource_id_from_env_with_reason, + ) + + app_insights_resource_id, resource_reason = ( + resolve_appinsights_resource_id_from_env_with_reason() + ) + except Exception as exc: # noqa: BLE001 + resource_reason = f"Foundry App Insights metadata discovery failed: {exc}" + + if app_insights_resource_id: + diagnostics["target_source"] = "foundry_project_connection" + + if not app_insights_resource_id and not config.log_analytics_workspace_id: application_id, source, reason = _resolve_application_id() if application_id: diagnostics["target"] = application_id @@ -127,8 +144,9 @@ def collect_azure_monitor( "is configured, and no App Insights ApplicationId could be " "discovered from the connection string or Foundry project" ) - if reason: - diagnostics["discovery_reason"] = reason + discovery_reason = reason or resource_reason + if discovery_reason: + diagnostics["discovery_reason"] = discovery_reason return AzureMonitorPayload(diagnostics=diagnostics) try: @@ -146,7 +164,7 @@ def collect_azure_monitor( from ._credentials import format_source_error, get_shared_credential, log_source_error # noqa: F401 workspace_or_resource = ( - config.log_analytics_workspace_id or config.app_insights_resource_id + config.log_analytics_workspace_id or app_insights_resource_id ) diagnostics["target"] = workspace_or_resource @@ -175,7 +193,7 @@ def collect_azure_monitor( ) return AzureMonitorPayload(diagnostics=diagnostics) response = query_resource( - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=kql, timespan=None, ) @@ -224,7 +242,7 @@ def collect_azure_monitor( ) else: safety_response = client.query_resource( # type: ignore[union-attr] - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=safety_kql, timespan=None, ) @@ -265,7 +283,7 @@ def collect_azure_monitor( ) else: tok_response = client.query_resource( # type: ignore[union-attr] - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=token_kql, timespan=None, ) @@ -300,7 +318,7 @@ def collect_azure_monitor( ) else: rl_response = client.query_resource( # type: ignore[union-attr] - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=rl_kql, timespan=None, ) diff --git a/src/agentops/cli/app.py b/src/agentops/cli/app.py index 539d72ed..5eacc950 100644 --- a/src/agentops/cli/app.py +++ b/src/agentops/cli/app.py @@ -79,16 +79,6 @@ "for the manual." ) ) -telemetry_app = typer.Typer( - help="Import Azure Monitor telemetry into AgentOps datasets." -) -dashboard_app = typer.Typer( - help=( - "Deploy, open, and export the Foundry operations Azure Monitor " - "workbook (capacity, traffic and tokens, latency, errors, agent behavior)." - ) -) -telemetry_app.add_typer(dashboard_app, name="dashboard") app.add_typer(eval_app, name="eval") app.add_typer(report_app, name="report") app.add_typer(workflow_app, name="workflow") @@ -100,7 +90,6 @@ app.add_typer(init_app, name="init") app.add_typer(assert_app, name="assert") app.add_typer(redteam_app, name="redteam") -app.add_typer(telemetry_app, name="telemetry") log = get_logger(__name__) DEFAULT_REPORT_INPUT = Path(".agentops/results/latest/results.json") @@ -829,85 +818,6 @@ class ExplainPage: outputs=("`.github/skills/agentops-*` for Copilot", "`.claude/commands/agentops-*` for Claude Code"), examples=("agentops skills install", "agentops skills install --platform copilot", "agentops skills install --from github:org/repo@v1"), ), - ("telemetry", "dashboard"): ExplainPage( - title="Foundry operations dashboard", - command="agentops telemetry dashboard", - synopsis=("agentops telemetry dashboard COMMAND [ARGS]...", "agentops telemetry dashboard explain"), - summary=( - "Deploys, opens, and exports the Foundry operations Azure Monitor " - "workbook: capacity (PTU), traffic and tokens, latency percentiles, " - "errors and throttling, and read-only Foundry trace-evaluation " - "behavior for an Azure OpenAI resource.", - "The workbook is scoped per Azure OpenAI resource and per Log " - "Analytics workspace and reads from AzureMetrics, AzureDiagnostics, " - "and Foundry-owned Application Insights events and spans.", - ), - children=("deploy", "open", "export"), - ), - ("telemetry", "dashboard", "deploy"): ExplainPage( - title="Deploy the Foundry operations workbook", - command="agentops telemetry dashboard deploy", - synopsis=( - "agentops telemetry dashboard deploy [--dry-run] [--subscription ID] " - "[--resource-group RG] [--workspace-id ID] [--name NAME] [--dir PATH]", - "agentops telemetry dashboard deploy explain", - ), - summary=( - "Deploys the workbook as a Microsoft.Insights/workbooks ARM resource " - "into the discovered (or supplied) resource group.", - "This is the first AgentOps CLI command that creates an Azure " - "resource; it deploys a single workbook and nothing else.", - ), - how_it_works=( - "Discovers subscription, resource group, Log Analytics workspace, " - "and the Azure OpenAI resource from agentops.yaml and the azd env.", - "Runs an RBAC preflight: Workbook Contributor on the resource group " - "and Log Analytics Reader on the workspace. Missing roles fail with " - "the exact role and scope to request.", - "Warns (non-fatally) and prints the exact " - "`az monitor diagnostic-settings create` command when the Azure " - "OpenAI resource is missing the RequestResponse or " - "AzureOpenAIRequestUsage categories.", - "Deploys via `az deployment group create` and prints the portal URL.", - ), - outputs=("A deployed workbook and its Azure portal URL", "The ARM template when --dry-run is used"), - examples=( - "agentops telemetry dashboard deploy --dry-run", - "agentops telemetry dashboard deploy --resource-group my-rg", - ), - ), - ("telemetry", "dashboard", "open"): ExplainPage( - title="Open the Foundry operations workbook", - command="agentops telemetry dashboard open", - synopsis=( - "agentops telemetry dashboard open [--print-url] [--subscription ID] " - "[--resource-group RG] [--name NAME] [--dir PATH]", - "agentops telemetry dashboard open explain", - ), - summary=( - "Builds the Azure portal URL for the workbook and opens it in the " - "default browser.", - "In a non-interactive shell, or with --print-url, it prints the URL " - "instead of opening a browser.", - ), - examples=( - "agentops telemetry dashboard open", - "agentops telemetry dashboard open --print-url", - ), - ), - ("telemetry", "dashboard", "export"): ExplainPage( - title="Export the workbook JSON", - command="agentops telemetry dashboard export", - synopsis=( - "agentops telemetry dashboard export [--out PATH]", - "agentops telemetry dashboard export explain", - ), - summary=( - "Copies the packaged workbook JSON to a local path so you can import " - "it manually or customize it before deploying.", - ), - examples=("agentops telemetry dashboard export --out foundry-ops.workbook.json",), - ), ("mcp",): ExplainPage( title="MCP commands", command="agentops mcp", @@ -1589,7 +1499,6 @@ def _cmd( prompt_app.command("explain")(_make_group_explain(("prompt",))) mcp_app.command("explain")(_make_group_explain(("mcp",))) agent_app.command("explain")(_make_group_explain(("agent",))) -dashboard_app.command("explain")(_make_group_explain(("telemetry", "dashboard"))) # --------------------------------------------------------------------------- @@ -2485,306 +2394,12 @@ def cmd_eval_promote_traces( ) -@telemetry_app.command("validate") -def cmd_telemetry_validate( - name: Annotated[str, typer.Argument(help="Name under telemetry_imports.")], - config: Annotated[ - Optional[Path], - typer.Option("--config", "-c", help="Path to agentops.yaml."), - ] = None, -) -> None: - """Validate a named telemetry import without querying Azure.""" - - from agentops.core.config_loader import load_agentops_config - from agentops.services.telemetry_import import ( - TelemetryImportError, - find_telemetry_import, - validate_telemetry_import, - ) - - try: - cfg = load_agentops_config(_resolve_eval_config_path(config)) - item = find_telemetry_import(cfg, name) - warnings = validate_telemetry_import(item) - except (TelemetryImportError, ValueError) as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(1) from exc - typer.echo(_cli_ok(f"telemetry import {name!r} is valid")) - for warning in warnings: - typer.echo(_cli_warn(f"warning: {warning}")) - - -@telemetry_app.command("preview") -def cmd_telemetry_preview( - name: Annotated[str, typer.Argument(help="Name under telemetry_imports.")], - rows: Annotated[int, typer.Option("--rows", min=1, help="Maximum rows to preview.")] = 10, - config: Annotated[ - Optional[Path], - typer.Option("--config", "-c", help="Path to agentops.yaml."), - ] = None, -) -> None: - """Query Azure Monitor and print a small dataset preview.""" - - from agentops.core.config_loader import load_agentops_config - from agentops.services.telemetry_import import ( - TelemetryImportError, - find_telemetry_import, - preview_telemetry_import, - render_telemetry_import_preview, - ) - - try: - cfg = load_agentops_config(_resolve_eval_config_path(config)) - item = find_telemetry_import(cfg, name) - preview = preview_telemetry_import(item, rows=rows, apply=False) - except (TelemetryImportError, ValueError) as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(1) from exc - typer.echo(render_telemetry_import_preview(preview)) - - -@telemetry_app.command("import") -def cmd_telemetry_import( - name: Annotated[str, typer.Argument(help="Name under telemetry_imports.")], - apply: Annotated[ - bool, - typer.Option("--apply", help="Write JSONL rows and manifest."), - ] = False, - rows: Annotated[ - Optional[int], - typer.Option("--rows", min=1, help="Optional maximum rows to import."), - ] = None, - config: Annotated[ - Optional[Path], - typer.Option("--config", "-c", help="Path to agentops.yaml."), - ] = None, -) -> None: - """Import telemetry into the configured JSONL output path.""" - - from agentops.core.config_loader import load_agentops_config - from agentops.services.telemetry_import import ( - TelemetryImportError, - find_telemetry_import, - preview_telemetry_import, - render_telemetry_import_preview, - ) - - if not apply: - typer.echo( - _cli_warn( - "Dry run only. Re-run with --apply to write the JSONL dataset and manifest." - ) - ) - try: - cfg = load_agentops_config(_resolve_eval_config_path(config)) - item = find_telemetry_import(cfg, name) - preview = preview_telemetry_import(item, rows=rows, apply=apply) - except (TelemetryImportError, ValueError) as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(1) from exc - typer.echo(render_telemetry_import_preview(preview)) - if apply: - typer.echo(_cli_updated(preview.output_path)) - typer.echo(_cli_updated(preview.manifest_path)) - - def _resolve_eval_config_path(config: Path | None) -> Path: if config is not None: return config return Path("agentops.yaml") -# --------------------------------------------------------------------------- -# agentops telemetry dashboard {deploy, open, export} -# --------------------------------------------------------------------------- -@dashboard_app.command("deploy") -def cmd_dashboard_deploy( - dry_run: Annotated[ - bool, - typer.Option("--dry-run", help="Emit the ARM template and make no changes."), - ] = False, - subscription: Annotated[ - Optional[str], - typer.Option("--subscription", help="Azure subscription id override."), - ] = None, - resource_group: Annotated[ - Optional[str], - typer.Option("--resource-group", help="Resource group for the workbook."), - ] = None, - workspace_id: Annotated[ - Optional[str], - typer.Option("--workspace-id", help="Log Analytics workspace resource id."), - ] = None, - name: Annotated[ - Optional[str], - typer.Option("--name", help="Workbook display name."), - ] = None, - workspace: Annotated[ - Path, - typer.Option("--dir", help="AgentOps workspace root for discovery."), - ] = Path("."), - explain: Annotated[str | None, typer.Argument(hidden=True)] = None, -) -> None: - """Deploy the Foundry operations workbook to Azure Monitor.""" - - if _maybe_explain_leaf(("telemetry", "dashboard", "deploy"), explain): - return - - import json - - from agentops.services import dashboard as dash - - target = dash.discover_target( - workspace.resolve(), - subscription_id=subscription, - resource_group=resource_group, - workspace_id=workspace_id, - name=name, - ) - - if dry_run: - template = dash.build_arm_template(target=target) - typer.echo(json.dumps(template, indent=2)) - typer.echo( - _cli_warn( - "Dry run only. No Azure changes were made. Re-run without " - "--dry-run to deploy." - ), - err=True, - ) - return - - # RBAC preflight — fail gracefully with the exact role and scope needed. - rbac = dash.check_rbac( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - workspace_id=target.workspace_id, - ) - for message in rbac.messages: - if rbac.level == "ok": - typer.echo(_cli_ok(message)) - elif rbac.level == "warn": - typer.echo(_cli_warn(message), err=True) - else: - typer.echo(_cli_error(message), err=True) - if not rbac.ok: - raise typer.Exit(code=1) - - # Diagnostic-settings advisory (non-fatal): print the exact fix command. - enabled = list(target.discovery.get("enabled_log_categories", []) or []) - missing = dash.missing_diagnostic_categories(enabled) if enabled else list( - dash.REQUIRED_DIAGNOSTIC_CATEGORIES - ) - if missing: - typer.echo( - _cli_warn( - "The Azure OpenAI resource may not emit the categories the " - f"workbook needs ({', '.join(missing)}). If the tiles are " - "empty, enable them with:" - ), - err=True, - ) - typer.echo( - _cli_command( - dash.build_diagnostic_settings_command( - aoai_resource_id=target.aoai_resource_id, - workspace_id=target.workspace_id, - ) - ), - err=True, - ) - - try: - url = dash.deploy_workbook(target=target) - except dash.DashboardError as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(code=1) from exc - - typer.echo(_cli_ok("Workbook deployed.")) - typer.echo(f"{_cli_label('Portal')}: {_cli_path(url)}") - - -@dashboard_app.command("open") -def cmd_dashboard_open( - print_url: Annotated[ - bool, - typer.Option("--print-url", help="Print the URL instead of opening a browser."), - ] = False, - subscription: Annotated[ - Optional[str], - typer.Option("--subscription", help="Azure subscription id override."), - ] = None, - resource_group: Annotated[ - Optional[str], - typer.Option("--resource-group", help="Resource group for the workbook."), - ] = None, - name: Annotated[ - Optional[str], - typer.Option("--name", help="Workbook display name."), - ] = None, - workspace: Annotated[ - Path, - typer.Option("--dir", help="AgentOps workspace root for discovery."), - ] = Path("."), - explain: Annotated[str | None, typer.Argument(hidden=True)] = None, -) -> None: - """Open the Foundry operations workbook in the Azure portal.""" - - if _maybe_explain_leaf(("telemetry", "dashboard", "open"), explain): - return - - from agentops.services import dashboard as dash - - target = dash.discover_target( - workspace.resolve(), - subscription_id=subscription, - resource_group=resource_group, - name=name, - ) - url = dash.build_workbook_portal_url( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - name=target.name, - tenant_id=target.tenant_id, - ) - - if print_url or not _stream_is_interactive(sys.stdout): - typer.echo(url) - return - typer.echo(f"{_cli_heading('Foundry operations dashboard')} → {_cli_path(url)}") - try: - webbrowser.open(url) - except Exception: # noqa: BLE001 - best effort - typer.echo(url) - - -@dashboard_app.command("export") -def cmd_dashboard_export( - out: Annotated[ - Path, - typer.Option("--out", help="Destination path for the workbook JSON."), - ] = Path("foundry-ops.workbook.json"), - explain: Annotated[str | None, typer.Argument(hidden=True)] = None, -) -> None: - """Export the packaged workbook JSON to a local path.""" - - if _maybe_explain_leaf(("telemetry", "dashboard", "export"), explain): - return - - from agentops.services import dashboard as dash - - try: - content = dash.load_workbook_template() - except dash.DashboardError as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(code=1) from exc - - destination = out.resolve() - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - typer.echo(_cli_updated(destination)) - - def _append_assert_step_summary(result, *, scored_cases, pass_rate) -> None: """Append an ASSERT gate summary to the GitHub Actions step summary.""" from agentops.core.step_summary import append_step_summary, is_active diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py index 2728a374..1276b91d 100644 --- a/src/agentops/core/agentops_config.py +++ b/src/agentops/core/agentops_config.py @@ -85,11 +85,6 @@ #: Where the local evaluator runtime gets the response text for each row. ResponseSource = Literal["agent", "dataset"] -#: Production telemetry import providers and destinations. -TelemetrySourceProvider = Literal["azure-monitor"] -TelemetryTarget = Literal["application-insights", "log-analytics"] -TelemetryLabelMode = Literal["self-similarity", "pending"] - #: Internal-only literal kept for the publisher dispatch table. Derived from #: ``execution`` + ``publish`` via :meth:`AgentOpsConfig.publish_target`. PublishTarget = Literal["foundry", "foundry_cloud"] @@ -391,116 +386,6 @@ def _url_non_empty(cls, value: Optional[str]) -> Optional[str]: return value -# --------------------------------------------------------------------------- -# Telemetry import configuration -# --------------------------------------------------------------------------- - - -class TelemetryTimeRangeConfig(BaseModel): - """Time window for a telemetry import query. - - Users can either provide explicit ISO-ish ``from``/``to`` timestamps or a - relative ``lookback_days`` window. The service owns final KQL rendering so - users never pass arbitrary query text. - """ - - from_: Optional[str] = Field(None, alias="from") - to: Optional[str] = None - lookback_days: Optional[int] = Field(None, ge=1, le=90) - - model_config = ConfigDict(extra="forbid", populate_by_name=True) - - @model_validator(mode="after") - def _validate_window(self) -> "TelemetryTimeRangeConfig": - explicit = self.from_ is not None or self.to is not None - if explicit and not (self.from_ and self.to): - raise ValueError("telemetry_imports.time_range requires both from and to") - if explicit and self.lookback_days is not None: - raise ValueError("telemetry_imports.time_range cannot mix from/to with lookback_days") - if not explicit and self.lookback_days is None: - self.lookback_days = 7 - return self - - -class TelemetryPrivacyConfig(BaseModel): - """Privacy controls applied before JSONL rows are written.""" - - redact_fields: List[str] = Field( - default_factory=lambda: ["authorization", "api_key", "token", "password", "secret"], - description="Case-insensitive field-name fragments to redact.", - ) - max_field_length: int = Field(4000, ge=100, le=20000) - include_raw: bool = False - - model_config = ConfigDict(extra="forbid") - - -class TelemetryOutputConfig(BaseModel): - """Output paths and labeling mode for generated dataset rows.""" - - path: Path = Field(Path(".agentops") / "data" / "telemetry-import.jsonl") - manifest_path: Optional[Path] = None - label_mode: TelemetryLabelMode = "self-similarity" - - model_config = ConfigDict(extra="forbid") - - -class TelemetryImportConfig(BaseModel): - """Named telemetry import declaration. - - The MVP intentionally keeps this declarative: users choose a supported - source/destination pair, field mappings, filters, privacy settings, and an - output file. The service generates the KQL. - """ - - name: str - source: TelemetrySourceProvider = "azure-monitor" - target: TelemetryTarget - resource_id: Optional[str] = None - workspace_id: Optional[str] = None - application_id: Optional[str] = None - connection_string: Optional[str] = None - time_range: TelemetryTimeRangeConfig = Field(default_factory=TelemetryTimeRangeConfig) - filters: Dict[str, str | List[str]] = Field(default_factory=dict) - fields: Dict[str, str] = Field(default_factory=dict) - privacy: TelemetryPrivacyConfig = Field(default_factory=TelemetryPrivacyConfig) - output: TelemetryOutputConfig = Field(default_factory=TelemetryOutputConfig) - max_rows: int = Field(100, ge=1, le=5000) - - model_config = ConfigDict(extra="forbid") - - @field_validator("name") - @classmethod - def _name_non_empty(cls, value: str) -> str: - value = value.strip() - if not value: - raise ValueError("telemetry_imports.name must be non-empty") - return value - - @field_validator("resource_id", "workspace_id", "application_id", "connection_string") - @classmethod - def _optional_text_non_empty(cls, value: Optional[str]) -> Optional[str]: - if value is None: - return value - value = value.strip() - if not value: - raise ValueError("telemetry_imports resource identifiers must be non-empty") - return value - - @model_validator(mode="after") - def _validate_target_ids(self) -> "TelemetryImportConfig": - if self.target == "log-analytics" and not self.workspace_id: - raise ValueError("telemetry_imports targeting log-analytics require workspace_id") - if self.target == "application-insights" and not ( - self.resource_id or self.application_id or self.connection_string - ): - raise ValueError( - "telemetry_imports targeting application-insights require resource_id, " - "application_id, or connection_string" - ) - return self - - class PromptAgentBootstrap(BaseModel): """Bootstrap defaults for prompt-agent CI/CD when the target Foundry project does not yet contain the seed agent referenced by ``agent``. @@ -1048,10 +933,6 @@ class AgentOpsConfig(BaseModel): ) evaluators: Optional[List[EvaluatorOverride]] = None - telemetry_imports: List[TelemetryImportConfig] = Field( - default_factory=list, - description="Named Azure Monitor imports that generate AgentOps JSONL datasets.", - ) rubrics: List[RubricConfig] = Field( default_factory=list, description="Optional context-specific rubric evaluator definitions.", diff --git a/src/agentops/services/agent_identity.py b/src/agentops/services/agent_identity.py index 3c3f29b7..71f7f1a6 100644 --- a/src/agentops/services/agent_identity.py +++ b/src/agentops/services/agent_identity.py @@ -518,10 +518,15 @@ def _name_from_target(workspace: Path) -> Optional[str]: if not isinstance(agent, str): return None raw = agent.strip() - if not raw or "://" in raw: + if not raw: return None - name = raw.split(":", 1)[0].strip() - return name or None + try: + from agentops.core.agentops_config import classify_agent + + target = classify_agent(raw) + except ValueError: + return None + return target.name def resolve_registration_inputs( diff --git a/src/agentops/services/dashboard.py b/src/agentops/services/dashboard.py deleted file mode 100644 index e694b8e9..00000000 --- a/src/agentops/services/dashboard.py +++ /dev/null @@ -1,506 +0,0 @@ -"""Thin service layer for the Foundry operations Azure Monitor workbook. - -This module is the single place that knows how to: - -* load the packaged ``foundry-ops.workbook.json`` gallery template, -* build the Azure portal deep link for the deployed workbook, -* run the RBAC / diagnostic-settings preflight for ``agentops telemetry - dashboard deploy``, and -* wrap the workbook content into a ``Microsoft.Insights/workbooks`` ARM - template and (optionally) deploy it via the Azure CLI. - -Azure SDK / CLI access is kept lazy and fail-open so importing this module -never requires the management SDKs or a live Azure session. The CLI layer in -:mod:`agentops.cli.app` stays thin and delegates the Azure logic here. -""" - -from __future__ import annotations - -import json -import logging -import shutil -import subprocess -import tempfile -import uuid -from dataclasses import dataclass, field -from importlib.resources import files as _pkg_files -from pathlib import Path -from typing import Any, Dict, Iterable, List, Mapping, Optional - -log = logging.getLogger(__name__) - -_TEMPLATE_PACKAGE = "agentops.templates" -_WORKBOOK_RESOURCE_PATH = "workbooks/foundry-ops.workbook.json" - -#: ARM resource type for an Azure Monitor workbook. -WORKBOOK_RESOURCE_TYPE = "Microsoft.Insights/workbooks" - -#: Diagnostic log categories the Azure OpenAI resource must emit for the -#: workbook queries to return data. -REQUIRED_DIAGNOSTIC_CATEGORIES = ("RequestResponse", "AzureOpenAIRequestUsage") - -_PORTAL_BASE = "https://portal.azure.com" - -# Deterministic namespace so ``deploy`` and ``open`` agree on the workbook -# resource name without querying Azure. -_WORKBOOK_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_DNS, "agentops.foundry-ops.workbook") - -# Built-in Azure role definition GUIDs used by the preflight. A role that can -# write workbooks (Owner / Contributor / Workbook Contributor) satisfies the -# deploy requirement; a role that can read the workspace satisfies the query -# requirement. -_ROLE_WORKBOOK_CONTRIBUTOR = "e8ddcd69-c73f-4f9f-9844-4100522f16ad" -_ROLE_LOG_ANALYTICS_READER = "73c42c96-874c-492b-b04d-ab87d138a893" -_ROLE_LOG_ANALYTICS_CONTRIBUTOR = "92aaf0da-9dab-42b6-94a3-d43ce8d16293" -_ROLE_READER = "acdd72a7-3385-48ef-bd42-f606fba81ae7" -_ROLE_CONTRIBUTOR = "b24988ac-6180-42a0-ab88-20f7382dd24c" -_ROLE_OWNER = "8e3af657-a8ff-443c-a75c-2fe8c4bcb635" - -_WORKBOOK_WRITE_ROLES = frozenset( - {_ROLE_WORKBOOK_CONTRIBUTOR, _ROLE_CONTRIBUTOR, _ROLE_OWNER} -) -_WORKSPACE_READ_ROLES = frozenset( - { - _ROLE_LOG_ANALYTICS_READER, - _ROLE_LOG_ANALYTICS_CONTRIBUTOR, - _ROLE_READER, - _ROLE_CONTRIBUTOR, - _ROLE_OWNER, - } -) - - -class DashboardError(RuntimeError): - """Raised for user-facing dashboard failures (surfaced by the CLI).""" - - -@dataclass -class DashboardTarget: - """Resolved Azure context for the workbook deploy.""" - - name: str = "AgentOps Foundry operations" - subscription_id: Optional[str] = None - resource_group: Optional[str] = None - workspace_id: Optional[str] = None - aoai_resource_id: Optional[str] = None - aoai_account_name: Optional[str] = None - tenant_id: Optional[str] = None - location: Optional[str] = None - discovery: Dict[str, Any] = field(default_factory=dict) - - -@dataclass -class PreflightResult: - """Outcome of a single preflight check. - - ``ok`` is ``True`` when the check passed. ``level`` is ``"ok"``, - ``"warn"`` (deploy can still proceed) or ``"error"`` (deploy must stop). - ``messages`` carries friendly, actionable text for the CLI to print. - """ - - ok: bool - level: str - messages: List[str] = field(default_factory=list) - - -# --------------------------------------------------------------------------- -# Template loading -# --------------------------------------------------------------------------- -def load_workbook_template() -> str: - """Return the packaged workbook JSON as a string.""" - resource = _pkg_files(_TEMPLATE_PACKAGE).joinpath(_WORKBOOK_RESOURCE_PATH) - return resource.read_text(encoding="utf-8") - - -def load_workbook_content() -> Dict[str, Any]: - """Return the packaged workbook JSON parsed into a dict.""" - try: - return json.loads(load_workbook_template()) - except (OSError, json.JSONDecodeError) as exc: # pragma: no cover - packaging - raise DashboardError( - f"Could not load the packaged workbook template: {exc}" - ) from exc - - -# --------------------------------------------------------------------------- -# Portal URL -# --------------------------------------------------------------------------- -def make_workbook_resource_id( - subscription_id: str, resource_group: str, name: str -) -> str: - """Build the deterministic ARM id for the deployed workbook. - - The workbook resource name is a GUID derived from the display name and - resource group so ``deploy`` and ``open`` agree without a live lookup. - """ - guid = str(uuid.uuid5(_WORKBOOK_NAMESPACE, f"{resource_group}:{name}")) - return ( - f"/subscriptions/{subscription_id}" - f"/resourceGroups/{resource_group}" - f"/providers/{WORKBOOK_RESOURCE_TYPE}/{guid}" - ) - - -def build_workbook_portal_url( - *, - workbook_resource_id: Optional[str] = None, - tenant_id: Optional[str] = None, - subscription_id: Optional[str] = None, - resource_group: Optional[str] = None, - name: Optional[str] = None, -) -> str: - """Return an Azure portal deep link for the workbook. - - When the workbook ARM id is known (directly, or derivable from - subscription/resource-group/name) the link opens that workbook. Otherwise - it falls back to the Azure Monitor Workbooks gallery so the tile is never - broken. - """ - rid = workbook_resource_id - if not rid and subscription_id and resource_group and name: - rid = make_workbook_resource_id(subscription_id, resource_group, name) - if rid: - prefix = f"{_PORTAL_BASE}/#@{tenant_id}" if tenant_id else f"{_PORTAL_BASE}/#" - return f"{prefix}/resource{rid}/workbook" - return ( - f"{_PORTAL_BASE}/#view/Microsoft_Azure_Monitoring_Workbooks" - "/WorkbookMenuBlade/~/gallery" - ) - - -# --------------------------------------------------------------------------- -# Diagnostic settings -# --------------------------------------------------------------------------- -def missing_diagnostic_categories(enabled_categories: Iterable[str]) -> List[str]: - """Return the required categories not present in ``enabled_categories``.""" - enabled = {str(c) for c in enabled_categories} - return [c for c in REQUIRED_DIAGNOSTIC_CATEGORIES if c not in enabled] - - -def build_diagnostic_settings_command( - *, - aoai_resource_id: Optional[str], - workspace_id: Optional[str], - name: str = "agentops-foundry-ops", -) -> str: - """Return the exact ``az`` command that wires the required categories.""" - resource = aoai_resource_id or "" - workspace = workspace_id or "" - logs = json.dumps( - [{"category": c, "enabled": True} for c in REQUIRED_DIAGNOSTIC_CATEGORIES] - ) - return ( - "az monitor diagnostic-settings create " - f"--name {name} " - f"--resource {resource} " - f"--workspace {workspace} " - f"--logs '{logs}'" - ) - - -# --------------------------------------------------------------------------- -# RBAC preflight -# --------------------------------------------------------------------------- -def check_rbac( - *, - subscription_id: Optional[str], - resource_group: Optional[str], - workspace_id: Optional[str], -) -> PreflightResult: - """Preflight the caller's RBAC for a workbook deploy. - - Requires ``Workbook Contributor`` (or a superset) on the resource group and - ``Log Analytics Reader`` (or a superset) on the workspace. When the RBAC - listing cannot run (SDK missing, no credential, listing denied) the check - fails **open** with a warning so ``deploy`` can still be attempted and let - ARM enforce permissions. - """ - if not subscription_id or not resource_group: - return PreflightResult( - ok=False, - level="error", - messages=[ - "Cannot check RBAC: subscription and resource group are " - "unknown. Pass --subscription and --resource-group, or run " - "from an initialized AgentOps workspace.", - ], - ) - - rg_scope = f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}" - try: - from agentops.agent.checks._rbac_authorization import ( - AuthorizationCheckError, - list_principal_role_definition_ids, - resolve_signed_in_principal_object_id, - ) - except ImportError as exc: # pragma: no cover - shipped together - return PreflightResult( - ok=True, - level="warn", - messages=[ - "Skipping RBAC preflight (authorization helpers unavailable: " - f"{exc}). ARM will enforce permissions at deploy time.", - ], - ) - - try: - principal = resolve_signed_in_principal_object_id() - rg_roles = set( - list_principal_role_definition_ids( - subscription_id=subscription_id, - scope=rg_scope, - principal_object_id=principal, - ) - ) - except AuthorizationCheckError as exc: - return PreflightResult( - ok=True, - level="warn", - messages=[ - f"Skipping RBAC preflight ({exc}). ARM will enforce " - "permissions at deploy time.", - ], - ) - - messages: List[str] = [] - ok = True - if not (rg_roles & _WORKBOOK_WRITE_ROLES): - ok = False - messages.append( - "You need the 'Workbook Contributor' role on resource group " - f"'{resource_group}'. Ask an admin to grant it, or run with " - "--dry-run to emit the ARM template instead." - ) - - if workspace_id: - try: - ws_roles = set( - list_principal_role_definition_ids( - subscription_id=subscription_id, - scope=workspace_id, - principal_object_id=principal, - ) - ) - except AuthorizationCheckError as exc: - messages.append( - f"Could not verify Log Analytics access ({exc}); make sure you " - "have 'Log Analytics Reader' on the workspace." - ) - ws_roles = set() - if ws_roles and not (ws_roles & _WORKSPACE_READ_ROLES): - ok = False - messages.append( - "You need the 'Log Analytics Reader' role on workspace " - f"'{workspace_id}' so the workbook can query it." - ) - - if ok and not messages: - messages.append( - "RBAC preflight passed: you can write workbooks to " - f"'{resource_group}' and read the workspace." - ) - return PreflightResult(ok=ok, level="ok" if ok else "error", messages=messages) - - -# --------------------------------------------------------------------------- -# Discovery -# --------------------------------------------------------------------------- -_WORKSPACE_ENV_KEYS = ( - "AZURE_LOG_ANALYTICS_WORKSPACE_ID", - "AZURE_LOG_ANALYTICS_WORKSPACE_RESOURCE_ID", - "LOG_ANALYTICS_WORKSPACE_ID", - "AZURE_MONITOR_WORKSPACE_ID", - "AZURE_MONITOR_WORKSPACE_RESOURCE_ID", -) -_ACCOUNT_ENV_KEYS = ( - "AZURE_OPENAI_RESOURCE", - "AZURE_OPENAI_RESOURCE_NAME", - "AZURE_AI_SERVICES_RESOURCE_NAME", - "AZURE_AI_SERVICES_NAME", -) -_TENANT_ENV_KEYS = ("AZURE_TENANT_ID",) - - -def _first(values: Mapping[str, str], keys: Iterable[str]) -> Optional[str]: - for key in keys: - val = values.get(key) - if val: - return val - return None - - -def discover_target( - workspace: Optional[Path], - *, - subscription_id: Optional[str] = None, - resource_group: Optional[str] = None, - workspace_id: Optional[str] = None, - name: Optional[str] = None, -) -> DashboardTarget: - """Resolve the deploy target from explicit flags, then the AZD env. - - Reuses the AgentOps foundry discovery path (the AZD ``.env`` read used by - the Azure resources doctor source). Explicit arguments always win. - """ - env_values: Dict[str, str] = {} - discovery: Dict[str, Any] = {} - try: - from agentops.agent.sources.azure_resources import ( - _discover_azd_environment, - ) - - _env_name, env_values, azd_diag = _discover_azd_environment(workspace) - discovery["azd"] = azd_diag - except Exception as exc: # noqa: BLE001 - discovery is best-effort - discovery["azd"] = {"status": "error", "reason": str(exc)} - - sub = subscription_id or env_values.get("AZURE_SUBSCRIPTION_ID") - rg = resource_group or env_values.get("AZURE_RESOURCE_GROUP") - ws = workspace_id or _first(env_values, _WORKSPACE_ENV_KEYS) - tenant = _first(env_values, _TENANT_ENV_KEYS) - account = _first(env_values, _ACCOUNT_ENV_KEYS) - - aoai_resource_id: Optional[str] = None - if account and account.startswith("/subscriptions/"): - aoai_resource_id = account - account = account.rstrip("/").rsplit("/", 1)[-1] - elif account and sub and rg: - aoai_resource_id = ( - f"/subscriptions/{sub}/resourceGroups/{rg}" - f"/providers/Microsoft.CognitiveServices/accounts/{account}" - ) - - return DashboardTarget( - name=name or "AgentOps Foundry operations", - subscription_id=sub, - resource_group=rg, - workspace_id=ws, - aoai_resource_id=aoai_resource_id, - aoai_account_name=account, - tenant_id=tenant, - discovery=discovery, - ) - - -# --------------------------------------------------------------------------- -# ARM template + deploy -# --------------------------------------------------------------------------- -def build_arm_template( - *, - target: DashboardTarget, - workbook_content: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """Wrap the workbook content into a deployable ARM template.""" - content = ( - workbook_content if workbook_content is not None else load_workbook_content() - ) - guid = str( - uuid.uuid5(_WORKBOOK_NAMESPACE, f"{target.resource_group}:{target.name}") - ) - resource: Dict[str, Any] = { - "type": WORKBOOK_RESOURCE_TYPE, - "apiVersion": "2022-04-01", - "name": guid, - # Workbooks reject "global"; resolve to the target region at deploy - # time. The RG-scoped deployment always has a valid location, and an - # explicit target.location (a real Azure region) still wins when set. - "location": target.location or "[resourceGroup().location]", - "kind": "shared", - "properties": { - "displayName": target.name, - "serializedData": json.dumps(content), - "version": "1.0", - "sourceId": target.workspace_id or "Azure Monitor", - "category": "workbook", - }, - } - return { - "$schema": ( - "https://schema.management.azure.com/schemas/2019-04-01/" - "deploymentTemplate.json#" - ), - "contentVersion": "1.0.0.0", - "resources": [resource], - "outputs": { - "workbookId": { - "type": "string", - "value": f"[resourceId('{WORKBOOK_RESOURCE_TYPE}', '{guid}')]", - } - }, - } - - -def _az_executable() -> Optional[str]: - return shutil.which("az") or shutil.which("az.cmd") - - -def deploy_workbook( - *, - target: DashboardTarget, - workbook_content: Optional[Dict[str, Any]] = None, - timeout: int = 300, -) -> str: - """Deploy the workbook via ``az deployment group create``. - - Returns the portal URL for the deployed workbook. Raises - :class:`DashboardError` with a friendly message on any failure. - """ - if not target.subscription_id or not target.resource_group: - raise DashboardError( - "Deploy needs a subscription and resource group. Pass " - "--subscription and --resource-group, or run from an initialized " - "AgentOps workspace." - ) - az = _az_executable() - if az is None: - raise DashboardError( - "The Azure CLI ('az') was not found on PATH. Install it and run " - "'az login', or use --dry-run to emit the ARM template." - ) - - template = build_arm_template(target=target, workbook_content=workbook_content) - with tempfile.TemporaryDirectory(prefix="agentops-workbook-") as tmp: - template_path = Path(tmp) / "foundry-ops.deploy.json" - template_path.write_text(json.dumps(template), encoding="utf-8") - cmd = [ - az, - "deployment", - "group", - "create", - "--subscription", - target.subscription_id, - "--resource-group", - target.resource_group, - "--name", - "agentops-foundry-ops", - "--template-file", - str(template_path), - "--only-show-errors", - "--output", - "json", - ] - try: - completed = subprocess.run( # noqa: S603 - args are controlled - cmd, - capture_output=True, - text=True, - timeout=timeout, - check=False, - ) - except (OSError, subprocess.TimeoutExpired) as exc: - raise DashboardError( - f"Failed to run 'az deployment group create': {exc}" - ) from exc - - if completed.returncode != 0: - detail = (completed.stderr or completed.stdout or "").strip() - raise DashboardError( - f"Workbook deployment failed. The Azure CLI reported:\n{detail}" - ) - - return build_workbook_portal_url( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - name=target.name, - tenant_id=target.tenant_id, - ) diff --git a/src/agentops/services/telemetry_import.py b/src/agentops/services/telemetry_import.py deleted file mode 100644 index 286778c5..00000000 --- a/src/agentops/services/telemetry_import.py +++ /dev/null @@ -1,550 +0,0 @@ -"""Import Azure Monitor telemetry into AgentOps JSONL datasets. - -The module has two halves: - -* a pure transformer that maps telemetry rows into AgentOps dataset rows -* a thin Azure Monitor query wrapper with lazy SDK imports - -Users never provide raw KQL. The query builder only accepts structured time -ranges, field mappings, filters, and row limits from ``agentops.yaml``. -""" - -from __future__ import annotations - -import json -import os -import re -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Iterable, Optional - -from agentops.core.agentops_config import AgentOpsConfig, TelemetryImportConfig - -DEFAULT_MAX_ROWS = 100 -MAX_ROWS_CAP = 5000 - -_DEFAULT_FIELD_CANDIDATES: dict[str, tuple[str, ...]] = { - "input": ( - "input", - "query", - "prompt", - "message", - "user_message", - "customDimensions.input", - "customDimensions.query", - "customDimensions.prompt", - "customDimensions.gen_ai.prompt", - ), - "response": ( - "response", - "prediction", - "output", - "answer", - "completion", - "assistant_message", - "customDimensions.response", - "customDimensions.prediction", - "customDimensions.output", - "customDimensions.gen_ai.completion", - ), - "context": ( - "context", - "retrieved_context", - "grounding", - "customDimensions.context", - "customDimensions.retrieved_context", - "customDimensions.grounding", - ), - "retrieved_context_items": ( - "retrieved_context_items", - "context_items", - "customDimensions.retrieved_context_items", - "customDimensions.context_items", - ), - "tool_calls": ("tool_calls", "customDimensions.tool_calls"), - "trace_id": ("trace_id", "operation_Id", "operationId"), - "turn_id": ("turn_id", "span_id", "id", "customDimensions.turn_id"), - "timestamp": ("timestamp", "TimeGenerated", "time"), -} - -_QUERY_COLUMNS = ( - "timestamp", - "operation_Id = column_ifexists('operation_Id', '')", - "operationId = column_ifexists('operationId', '')", - "id = column_ifexists('id', '')", - "name = column_ifexists('name', '')", - "message = column_ifexists('message', '')", - "duration = column_ifexists('duration', '')", - "success = column_ifexists('success', '')", - "customDimensions = column_ifexists('customDimensions', dynamic({}))", -) - - -class TelemetryImportError(RuntimeError): - """Raised when a telemetry import cannot be validated, queried, or written.""" - - -@dataclass(frozen=True) -class TelemetryImportPreview: - """Result of validating/querying/transforming one telemetry import.""" - - config: TelemetryImportConfig - output_path: Path - manifest_path: Path - rows: list[dict[str, Any]] - skipped: int = 0 - deduped: int = 0 - truncated: bool = False - warnings: list[str] = field(default_factory=list) - - -def find_telemetry_import( - config: AgentOpsConfig, - name: str, -) -> TelemetryImportConfig: - """Return a named telemetry import or raise a friendly error.""" - - for item in config.telemetry_imports: - if item.name == name: - return item - available = ", ".join(item.name for item in config.telemetry_imports) or "none" - raise TelemetryImportError( - f"telemetry import {name!r} was not found in agentops.yaml. " - f"Available imports: {available}." - ) - - -def validate_telemetry_import(config: TelemetryImportConfig) -> list[str]: - """Validate service-level constraints and return non-fatal warnings.""" - - warnings: list[str] = [] - if config.output.label_mode == "self-similarity": - warnings.append( - "Generated rows use production responses as expected values for drift " - "detection, not human-verified ground truth." - ) - return warnings - - -def preview_telemetry_import( - config: TelemetryImportConfig, - *, - rows: Optional[int] = None, - apply: bool = False, -) -> TelemetryImportPreview: - """Query Azure Monitor, transform rows, and optionally write JSONL output.""" - - validate_telemetry_import(config) - raw_rows = query_azure_monitor(config, rows=rows) - preview = transform_telemetry_rows(config, raw_rows, rows=rows) - if apply: - write_telemetry_import(preview) - return preview - - -def transform_telemetry_rows( - config: TelemetryImportConfig, - telemetry_rows: Iterable[dict[str, Any]], - *, - rows: Optional[int] = None, -) -> TelemetryImportPreview: - """Pure transformation from telemetry records to AgentOps dataset rows.""" - - limit = _bounded_rows(rows if rows is not None else config.max_rows) - output_path = config.output.path - manifest_path = config.output.manifest_path or output_path.with_name( - f"{output_path.stem}-manifest.json" - ) - warnings = validate_telemetry_import(config) - converted: list[dict[str, Any]] = [] - skipped = 0 - deduped = 0 - seen: set[tuple[str, str]] = set() - - for raw in telemetry_rows: - if len(converted) >= limit: - break - row = _telemetry_row_to_agentops_row(config, raw) - if row is None: - skipped += 1 - continue - telemetry = row.get("telemetry") - trace_id = "" - turn_id = "" - if isinstance(telemetry, dict): - trace_id = str(telemetry.get("trace_id") or "") - turn_id = str(telemetry.get("turn_id") or "") - key = (trace_id or row["input"], turn_id or row.get("response", "")) - if key in seen: - deduped += 1 - continue - seen.add(key) - converted.append(row) - - truncated = len(converted) >= limit - if not converted: - warnings.append("No telemetry rows contained both input and response text.") - return TelemetryImportPreview( - config=config, - output_path=output_path, - manifest_path=manifest_path, - rows=converted, - skipped=skipped, - deduped=deduped, - truncated=truncated, - warnings=warnings, - ) - - -def write_telemetry_import(preview: TelemetryImportPreview) -> None: - """Write JSONL rows and a small manifest next to the output.""" - - preview.output_path.parent.mkdir(parents=True, exist_ok=True) - with preview.output_path.open("w", encoding="utf-8") as handle: - for row in preview.rows: - handle.write(json.dumps(row, ensure_ascii=False) + "\n") - - trace_ids = [ - str(row.get("telemetry", {}).get("trace_id")) - for row in preview.rows - if isinstance(row.get("telemetry"), dict) and row["telemetry"].get("trace_id") - ] - manifest = { - "version": 1, - "generated_at": datetime.now(timezone.utc).isoformat(), - "import": preview.config.name, - "source": preview.config.source, - "target": preview.config.target, - "output_path": str(preview.output_path), - "rows": len(preview.rows), - "skipped": preview.skipped, - "deduped": preview.deduped, - "truncated": preview.truncated, - "trace_ids": trace_ids, - "warnings": preview.warnings, - } - preview.manifest_path.write_text( - json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - - -def render_telemetry_import_preview(preview: TelemetryImportPreview) -> str: - """Render concise CLI output.""" - - lines = [ - "AgentOps telemetry import", - f"Import: {preview.config.name}", - f"Target: {preview.config.target}", - f"Output: {preview.output_path}", - "", - "Summary", - f" rows {len(preview.rows)}", - f" skipped {preview.skipped}", - f" deduped {preview.deduped}", - f" truncated {str(preview.truncated).lower()}", - ] - if preview.warnings: - lines.append("") - lines.append("Warnings") - lines.extend(f" - {warning}" for warning in preview.warnings) - if preview.rows: - lines.append("") - lines.append("Sample rows") - for index, row in enumerate(preview.rows[:3], start=1): - lines.append(f" {index}. {str(row.get('input', ''))[:120]}") - return "\n".join(lines) + "\n" - - -def query_azure_monitor( - config: TelemetryImportConfig, - *, - rows: Optional[int] = None, -) -> list[dict[str, Any]]: - """Run the generated KQL against Azure Monitor with lazy SDK imports.""" - - try: - from azure.identity import DefaultAzureCredential # noqa: WPS433 - except ImportError as exc: - raise TelemetryImportError( - "Telemetry import requires Azure authentication packages. Install " - "them with: python -m pip install azure-identity azure-monitor-query" - ) from exc - - kql = build_telemetry_kql(config, rows=rows) - credential = DefaultAzureCredential( - exclude_developer_cli_credential=True, - process_timeout=30, - ) - try: - if config.target == "log-analytics": - from azure.monitor.query import LogsQueryClient # noqa: WPS433 - - client = LogsQueryClient(credential) - workspace_id = _resolve_value(config.workspace_id, "workspace_id") - response = client.query_workspace(workspace_id, kql, timespan=None) - return _flatten_logs_response(response) - if config.resource_id: - from azure.monitor.query import LogsQueryClient # noqa: WPS433 - - client = LogsQueryClient(credential) - resource_id = _resolve_value(config.resource_id, "resource_id") - response = client.query_resource(resource_id, kql, timespan=None) - return _flatten_logs_response(response) - app_id = _application_id(config) - token = credential.get_token("https://api.applicationinsights.io/.default").token - return _query_application_insights(app_id, token, kql) - except ImportError as exc: - raise TelemetryImportError( - "Telemetry import with resource_id/workspace_id requires the Azure " - "Monitor Query SDK. Install it with: python -m pip install " - "azure-monitor-query" - ) from exc - except TelemetryImportError: - raise - except Exception as exc: # noqa: BLE001 - raise TelemetryImportError(f"Azure Monitor query failed: {exc}") from exc - - -def build_telemetry_kql( - config: TelemetryImportConfig, - *, - rows: Optional[int] = None, -) -> str: - """Build safe KQL from structured config only.""" - - limit = _bounded_rows(rows if rows is not None else config.max_rows) - clauses = ["union isfuzzy=true requests, dependencies, traces"] - clauses.append(f"| extend timestamp = {_timestamp_expr()}") - clauses.append(_time_clause(config)) - for key, value in sorted(config.filters.items()): - clauses.append(_filter_clause(key, value)) - columns = ", ".join(_QUERY_COLUMNS) - clauses.append(f"| project {columns}") - clauses.append("| order by timestamp desc") - clauses.append(f"| take {limit}") - return "\n".join(clauses) - - -def _telemetry_row_to_agentops_row( - config: TelemetryImportConfig, - raw: dict[str, Any], -) -> Optional[dict[str, Any]]: - input_text = _mapped_text(config, raw, "input") - response_text = _mapped_text(config, raw, "response") - if not input_text or not response_text: - return None - - label_mode = config.output.label_mode - telemetry = { - "trace_id": _mapped_text(config, raw, "trace_id"), - "turn_id": _mapped_text(config, raw, "turn_id"), - "timestamp": _mapped_text(config, raw, "timestamp"), - "source": config.source, - "target": config.target, - "import": config.name, - } - row: dict[str, Any] = { - "input": _clean_value(input_text, config), - "response": _clean_value(response_text, config), - "prediction": _clean_value(response_text, config), - "expected": _clean_value(response_text, config) if label_mode == "self-similarity" else "", - "telemetry": {k: v for k, v in telemetry.items() if v not in (None, "")}, - "metadata": { - "source": "azure_monitor_telemetry", - "label_mode": label_mode, - "needs_review": True, - }, - } - context = _mapped_value(config, raw, "context") - if context not in (None, "", [], {}): - row["context"] = _clean_value(context, config) - row["retrieved_context"] = row["context"] - context_items = _mapped_value(config, raw, "retrieved_context_items") - if context_items not in (None, "", [], {}): - row["retrieved_context_items"] = _clean_value(context_items, config) - tool_calls = _mapped_value(config, raw, "tool_calls") - if tool_calls not in (None, "", [], {}): - row["tool_calls"] = _clean_value(tool_calls, config) - if config.privacy.include_raw: - row["raw"] = _clean_value(raw, config) - return row - - -def _mapped_text(config: TelemetryImportConfig, raw: dict[str, Any], name: str) -> Optional[str]: - value = _mapped_value(config, raw, name) - if value is None: - return None - if isinstance(value, str): - value = value.strip() - return value or None - if isinstance(value, (dict, list)): - text = json.dumps(value, ensure_ascii=False) - return text if text not in ("{}", "[]") else None - text = str(value).strip() - return text or None - - -def _mapped_value(config: TelemetryImportConfig, raw: dict[str, Any], name: str) -> Any: - mapping = config.fields.get(name) - if mapping: - return _lookup(raw, mapping) - for candidate in _DEFAULT_FIELD_CANDIDATES.get(name, ()): - value = _lookup(raw, candidate) - if value not in (None, "", [], {}): - return value - return None - - -def _lookup(data: dict[str, Any], path: str) -> Any: - current: Any = data - for part in path.split("."): - if not isinstance(current, dict): - return None - current = current.get(part) - return current - - -def _clean_value(value: Any, config: TelemetryImportConfig, key: str = "") -> Any: - lowered = key.lower() - if any(fragment.lower() in lowered for fragment in config.privacy.redact_fields): - return "[redacted]" - if isinstance(value, dict): - return {k: _clean_value(v, config, str(k)) for k, v in value.items()} - if isinstance(value, list): - return [_clean_value(item, config, key) for item in value] - if isinstance(value, str) and len(value) > config.privacy.max_field_length: - return value[: config.privacy.max_field_length] + "...[truncated]" - return value - - -def _flatten_logs_response(response: Any) -> list[dict[str, Any]]: - tables = getattr(response, "tables", None) or [] - if not tables: - return [] - table = tables[0] - columns: list[str] = [] - for column in getattr(table, "columns", None) or []: - name = getattr(column, "name", None) if not isinstance(column, dict) else column.get("name") - if isinstance(name, str): - columns.append(name) - rows: list[dict[str, Any]] = [] - for raw in getattr(table, "rows", None) or []: - rows.append(dict(zip(columns, raw))) - return rows - - -def _application_id(config: TelemetryImportConfig) -> str: - if config.application_id: - return _resolve_value(config.application_id, "application_id") - if config.connection_string: - connection_string = _resolve_value(config.connection_string, "connection_string") - match = re.search(r"ApplicationId=([0-9a-fA-F-]+)", connection_string) - if match: - return match.group(1) - raise TelemetryImportError( - "application-insights imports require resource_id, application_id, or " - "a connection_string containing ApplicationId" - ) - - -def _query_application_insights(app_id: str, bearer: str, kql: str) -> list[dict[str, Any]]: - import json as _json - from urllib import request - - body = _json.dumps({"query": kql}).encode("utf-8") - req = request.Request( - url=f"https://api.applicationinsights.io/v1/apps/{app_id}/query", - data=body, - headers={ - "Authorization": f"Bearer {bearer}", - "Content-Type": "application/json", - }, - method="POST", - ) - with request.urlopen(req, timeout=30) as response: # noqa: S310 - parsed = _json.loads(response.read()) - if isinstance(parsed, dict) and parsed.get("error"): - err = parsed["error"] - message = err.get("message") if isinstance(err, dict) else str(err) - raise TelemetryImportError(f"Application Insights query failed: {message}") - tables = parsed.get("tables") if isinstance(parsed, dict) else None - if not tables: - return [] - table = tables[0] - columns = [column.get("name") for column in table.get("columns", [])] - return [dict(zip(columns, row)) for row in table.get("rows", [])] - - -def _time_clause(config: TelemetryImportConfig) -> str: - tr = config.time_range - if tr.from_ and tr.to: - return ( - f"| where timestamp between (datetime({_kql_string(tr.from_)}) .. " - f"datetime({_kql_string(tr.to)}))" - ) - days = tr.lookback_days or 7 - return f"| where timestamp >= ago({days}d)" - - -def _filter_clause(key: str, value: str | list[str]) -> str: - expr = _safe_column_expr(key) - values = value if isinstance(value, list) else [value] - escaped = ", ".join(_kql_string(str(item)) for item in values) - if len(values) == 1: - return f"| where {expr} == {escaped}" - return f"| where {expr} in ({escaped})" - - -def _safe_column_expr(key: str) -> str: - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?", key): - raise TelemetryImportError( - f"unsafe telemetry filter field {key!r}; use a column name or customDimensions.name" - ) - if key.startswith("customDimensions."): - subkey = key.split(".", 1)[1] - return ( - "tostring(column_ifexists('customDimensions', dynamic({}))" - f"[{_kql_string(subkey)}])" - ) - return f"tostring(column_ifexists({_kql_string(key)}, ''))" - - -def _timestamp_expr() -> str: - return ( - "coalesce(" - "column_ifexists('timestamp', datetime(null)), " - "column_ifexists('TimeGenerated', datetime(null)), " - "column_ifexists('time', datetime(null))" - ")" - ) - - -def _kql_string(value: str) -> str: - return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" - - -def _resolve_value(value: Optional[str], label: str) -> str: - if not value: - raise TelemetryImportError(f"telemetry import is missing {label}") - value = value.strip() - env_name: Optional[str] = None - if value.startswith("env:"): - env_name = value[4:] - elif value.startswith("$") and len(value) > 1: - env_name = value[1:].strip("{}") - if env_name: - resolved = os.getenv(env_name) - if not resolved: - raise TelemetryImportError( - f"environment variable {env_name} referenced by {label} is not set" - ) - return resolved - return value - - -def _bounded_rows(rows: int) -> int: - if rows <= 0: - raise TelemetryImportError("rows must be greater than zero") - return min(rows, MAX_ROWS_CAP) diff --git a/src/agentops/templates/workbooks/README.md b/src/agentops/templates/workbooks/README.md deleted file mode 100644 index 5ae54f53..00000000 --- a/src/agentops/templates/workbooks/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# Foundry operations dashboard (Azure Monitor workbook) - -`foundry-ops.workbook.json` is an Azure Monitor **gallery-template workbook** that -visualizes Azure AI Foundry / Azure OpenAI operational metrics end to end: PTU -utilization, PAYG spillover, traffic and token consumption, latency percentiles, -error / throttling rates, and Foundry-owned trace-evaluation results. - -The workbook is scoped **per Azure OpenAI resource** and **per Log Analytics -workspace**. Pick the subscription, workspace, and Azure OpenAI resource in the -parameters bar, then use the deployment, model, time-range, and streaming -filters to narrow each tab. - -## Tabs - -| Tab | Shows | -| --- | --- | -| Capacity | `PTU_Avg_Pct`, `PTU_Max_Pct`, `Ratelimit`, PAYG `Spillover`, `Aggregated_PTU_With_Spillover`, `PTU_Normalizado` | -| Traffic and tokens | `TotalCalls`, `AzureOpenAIRequests`, `InputTokens`, `OutputTokens`, `TotalTokens`, `ProcessedPromptTokens`, `GeneratedTokens`, `TokensPorRequest`, and a PTU vs PAYG spillover pivot | -| Latency | `TTFT`, `TBT`, `TTLT`, tokens/sec, time-to-response, and `P95` / `P99` / `Avg` / `Max` over `DurationMs` | -| Errors and throttling | HTTP `429` / `400` / `500` counts, `BlockedCalls`, `TasaThrottling_Pct`, `TasaError_Pct` | -| Agent behavior | Data status and freshness, observed `invoke_agent` invocations, evaluated traces, evaluation-event counts, per-evaluator pass rate / volume / raw-score trends, and recent failed or low-score trace correlation | - -`PTU_Normalizado` (`PTU% * 100000`) is a **display-only** scaling series so the -0-100 PTU utilization percentage can share a Y axis with raw token counts in a -single timechart. Read true utilization from `PTU_Avg_Pct` / `PTU_Max_Pct`. - -## KQL queries - -The raw KQL for each derived-metric family lives under -[`queries/`](./queries), so operators can reuse the logic outside the workbook: - -| File | Derived metrics | -| --- | --- | -| [`capacity_ptu_spillover.kql`](./queries/capacity_ptu_spillover.kql) | PTU avg/max %, Ratelimit, Spillover, Aggregated PTU with spillover, PTU_Normalizado | -| [`traffic_tokens.kql`](./queries/traffic_tokens.kql) | Calls, input/output/total tokens, processed/generated tokens, tokens per request, PTU vs PAYG split | -| [`latency_percentiles.kql`](./queries/latency_percentiles.kql) | TTFT/TBT/TTLT, tokens/sec, time-to-response, and P95/P99/Avg/Max over DurationMs | -| [`errors_throttling.kql`](./queries/errors_throttling.kql) | HTTP 429/400/500 counts, blocked calls, throttling rate, error rate | -| [`agent_behavior.kql`](./queries/agent_behavior.kql) | Versioned `agent_behavior/v1` normalization for Foundry evaluation events and observed `invoke_agent` spans across workspace-based and classic Application Insights tables | - -The standalone `.kql` files expose the parameter bindings as `let` declarations -at the top; the workbook substitutes the same values through its parameter bar. - -## Agent behavior tab - -The **Agent behavior** tab is an additive, read-only adapter over compatible -`gen_ai.evaluation.result` events. Current official Foundry documentation -explicitly verifies this event for -[human trace annotations](https://learn.microsoft.com/azure/foundry/observability/how-to/trace-annotations#log-end-user-feedback-as-trace-annotations). -It does not clearly guarantee that every automated Foundry trace-evaluation -result is exported through the same Application Insights event shape. -Automated event support is therefore **validation-dependent** and must be -proven in the target workspace. Trace evaluation and annotations are preview, -platform-owned features. AgentOps does not create, schedule, gate, or edit -them. - -The `agent_behavior/v1` KQL normalizer supports both recognized Application -Insights shapes: - -- workspace-based `AppEvents` (`Name`, `Properties`) and `AppDependencies` - (`Properties`); -- classic `customEvents` (`name`, `customDimensions`) and `dependencies` - (`customDimensions`). - -The mapping uses an explicit set of known `gen_ai.*` properties and keeps the -raw property bag in the normalized row. Missing optional values do not remove -events. Missing agent versions appear as **Version not reported**, and -nonnumeric score values remain visible as raw score text rather than becoming -zero. Trace-ID evaluation and correlation do not require `gen_ai.agent.id`; -agent and version filters simply use the metadata when a producer reports it. - -The status row appears before quality results and distinguishes these states: - -| State | Meaning | -| --- | --- | -| `Schema unavailable` | Matching event names exist, but none of the v1 evaluator, score, or label properties are recognized. Inspect the retained raw properties before updating the versioned mapping. | -| `No access` | The workbook query shows a native permission error. Request `Log Analytics Reader` on the selected workspace. | -| `No data` | The recognized event tables contain no compatible `gen_ai.evaluation.result` events in the selected time range. This can also mean automated trace evaluation does not export this event shape in the target environment. | -| `Filter empty` | Events exist in the time range, but none match the environment, agent, version, and evaluator filters. | -| `Possible ingestion delay` | The newest matching event is more than 15 minutes old. This may be ingestion delay or simply no recent Foundry trace evaluation. | - -The three count columns intentionally have different meanings: - -- **Observed invoke_agent invocations** counts distinct invocation keys visible - in the selected workspace. -- **Evaluated traces** counts distinct trace IDs represented by evaluation - events. -- **Evaluation events** counts evaluator results. One evaluated trace can emit - several events. - -The workbook does not calculate evaluation coverage because the workspace might -not contain a complete invocation denominator. Pass rate uses only recognized -pass/fail labels. Raw score trends stay in a table grouped by evaluator because -different evaluators can use different scales; pass-rate and event-volume charts -use comparable units. - -The recent-events table keeps trace, response, and conversation IDs when -available. Copy the trace ID, open the matching Microsoft Foundry project, -select **Tracing**, and search for the ID. The workbook cannot build a reliable -project-specific Foundry deep link from a Log Analytics workspace ID alone. - -## Required diagnostic settings - -The Azure OpenAI resource must send both of these log categories to the Log -Analytics workspace the workbook queries: - -- `RequestResponse` -- `AzureOpenAIRequestUsage` - -If they are missing, create them with: - -```bash -az monitor diagnostic-settings create \ - --name agentops-foundry-ops \ - --resource \ - --workspace \ - --logs '[{"category":"RequestResponse","enabled":true},{"category":"AzureOpenAIRequestUsage","enabled":true}]' -``` - -`agentops doctor` flags this automatically (rule -`waf.observability.aoai_diagnostic_categories`) and prints the exact command. - -## Required roles - -| Role | Scope | Why | -| --- | --- | --- | -| `Workbook Contributor` | Resource group that will hold the workbook | Deploy / update the `Microsoft.Insights/workbooks` resource | -| `Log Analytics Reader` | The Log Analytics workspace | Let the workbook query Azure OpenAI metrics/logs and Foundry-owned Application Insights events/spans | - -## How to use it - -Ship it with the CLI: - -```bash -# Emit the ARM template without touching Azure. -agentops telemetry dashboard deploy --dry-run - -# Deploy the workbook (needs Workbook Contributor on the resource group). -agentops telemetry dashboard deploy - -# Open the workbook in the Azure portal. -agentops telemetry dashboard open - -# Copy the workbook JSON to a local path for manual import. -agentops telemetry dashboard export --out ./foundry-ops.workbook.json -``` - -Or import it manually: **Azure portal → Monitor → Workbooks → New → Advanced -Editor**, paste the contents of `foundry-ops.workbook.json`, then Apply and Save, -selecting the target workspace and Azure OpenAI resource. - -## Authoring note (validate before GA) - -This workbook JSON was authored from the issue's KQL and the Azure Monitor -workbook schema **without a live Azure environment**. The Agent behavior -normalizer is validated with repository fixtures for both recognized table -shapes, missing optional fields, nonnumeric scores, multiple evaluators, absent -versions, and an unrecognized future schema. No live Foundry evaluation results -were invented or claimed. These synthetic fixtures validate the adapter; they -are not evidence that an automated Foundry trace-evaluation producer emits the -same event schema. - -Before relying on the workbook in production, smoke-validate it against a real -Log Analytics workspace that receives `RequestResponse`, -`AzureOpenAIRequestUsage`, Foundry `invoke_agent` spans, and -`gen_ai.evaluation.result` events. Confirm the parameters resolve, each tab -renders, data status is accurate, trace IDs correlate in Foundry Tracing, and -the derived metrics match expectations. Validate human-annotation and automated -trace-evaluation producers separately. Update the versioned normalizer instead -of silently accepting renamed properties. diff --git a/src/agentops/templates/workbooks/foundry-ops.workbook.json b/src/agentops/templates/workbooks/foundry-ops.workbook.json deleted file mode 100644 index 1af661a5..00000000 --- a/src/agentops/templates/workbooks/foundry-ops.workbook.json +++ /dev/null @@ -1,683 +0,0 @@ -{ - "version": "Notebook/1.0", - "items": [ - { - "type": 1, - "content": { - "json": "# Foundry operations dashboard\nPTU capacity, traffic and tokens, latency, errors, and read-only agent behavior for Azure AI Foundry / Azure OpenAI deployments." - }, - "name": "title" - }, - { - "type": 9, - "content": { - "version": "KqlParameterItem/1.0", - "parameters": [ - { - "id": "p-subscription", - "version": "KqlParameterItem/1.0", - "name": "Subscription", - "label": "Subscription", - "type": 6, - "isRequired": true, - "typeSettings": { - "additionalResourceOptions": [], - "includeAll": false - } - }, - { - "id": "p-workspace", - "version": "KqlParameterItem/1.0", - "name": "Workspace", - "label": "Log Analytics workspace", - "type": 5, - "isRequired": true, - "resourceType": "microsoft.operationalinsights/workspaces", - "typeSettings": { - "resourceTypeFilter": { - "microsoft.operationalinsights/workspaces": true - }, - "additionalResourceOptions": [] - } - }, - { - "id": "p-resource", - "version": "KqlParameterItem/1.0", - "name": "Resource", - "label": "Azure OpenAI resource", - "type": 5, - "isRequired": true, - "resourceType": "microsoft.cognitiveservices/accounts", - "typeSettings": { - "resourceTypeFilter": { - "microsoft.cognitiveservices/accounts": true - }, - "additionalResourceOptions": [] - } - }, - { - "id": "p-timerange", - "version": "KqlParameterItem/1.0", - "name": "TimeRange", - "label": "Time range", - "type": 4, - "isRequired": true, - "typeSettings": { - "selectableValues": [ - { - "durationMs": 3600000 - }, - { - "durationMs": 14400000 - }, - { - "durationMs": 43200000 - }, - { - "durationMs": 86400000 - }, - { - "durationMs": 604800000 - }, - { - "durationMs": 2592000000 - } - ] - }, - "value": { - "durationMs": 86400000 - } - }, - { - "id": "p-deployment", - "version": "KqlParameterItem/1.0", - "name": "Deployment", - "label": "Deployment", - "type": 2, - "isRequired": false, - "multiSelect": false, - "typeSettings": { - "additionalResourceOptions": [ - "value::all" - ], - "showDefault": false - }, - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName)\n| where isnotempty(Deployment)\n| distinct Deployment\n| order by Deployment asc", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "value": "value::all" - }, - { - "id": "p-model", - "version": "KqlParameterItem/1.0", - "name": "Model", - "label": "Model", - "type": 2, - "isRequired": false, - "multiSelect": false, - "typeSettings": { - "additionalResourceOptions": [ - "value::all" - ], - "showDefault": false - }, - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Model = tostring(props.modelName)\n| where isnotempty(Model)\n| distinct Model\n| order by Model asc", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "value": "value::all" - }, - { - "id": "p-streamtype", - "version": "KqlParameterItem/1.0", - "name": "StreamType", - "label": "Streaming", - "type": 2, - "isRequired": false, - "multiSelect": false, - "typeSettings": { - "additionalResourceOptions": [] - }, - "jsonData": "[{\"value\": \"*\", \"label\": \"All\"}, {\"value\": \"Streaming\", \"label\": \"Streaming\"}, {\"value\": \"NonStreaming\", \"label\": \"Non-streaming\"}]", - "value": "*" - } - ], - "style": "pills", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces" - }, - "name": "parameters - top" - }, - { - "type": 11, - "content": { - "version": "LinkItem/1.0", - "style": "tabs", - "links": [ - { - "id": "tab-capacity", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Capacity", - "subTarget": "capacity", - "style": "link" - }, - { - "id": "tab-traffic", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Traffic and tokens", - "subTarget": "traffic", - "style": "link" - }, - { - "id": "tab-latency", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Latency", - "subTarget": "latency", - "style": "link" - }, - { - "id": "tab-errors", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Errors and throttling", - "subTarget": "errors", - "style": "link" - }, - { - "id": "tab-agent-behavior", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Agent behavior", - "subTarget": "agent-behavior", - "style": "link" - } - ] - }, - "name": "tabs" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "capacity" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "## Foundry / Azure OpenAI operations dashboard\nThis workbook is scoped **per Azure OpenAI resource** and **per Log Analytics workspace**. Pick the subscription, workspace, and Azure OpenAI resource above, then use the deployment, model, time-range, and streaming filters to narrow each tab.\n\n> **PTU_Normalizado** is a display-only scaling series (`PTU% * 100000`). It exists so the 0-100 PTU utilization percentage can share a Y axis with raw token counts in a single timechart. It is not a real token volume - read the true utilization from **PTU_Avg_Pct** / **PTU_Max_Pct**." - }, - "name": "capacity-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureMetrics\n| where MetricName in ('AzureOpenAIProvisionedManagedUtilizationV2','Ratelimit','TokenTransaction','ProcessedPromptTokens')\n| extend Deployment = tostring(split(Resource, '/')[-1])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| summarize PTU_Avg_Pct = avgif(Average, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n PTU_Max_Pct = maxif(Maximum, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n Ratelimit = avgif(Average, MetricName == 'Ratelimit'),\n ProcessedTokens = sumif(Total, MetricName in ('TokenTransaction','ProcessedPromptTokens'))\n by TimeGenerated = bin(TimeGenerated, 1m), Deployment\n| extend Spillover = max_of(0.0, ProcessedTokens - Ratelimit)\n| extend Aggregated_PTU_With_Spillover = ProcessedTokens\n| extend PTU_Normalizado = PTU_Avg_Pct * 100000\n| project TimeGenerated, PTU_Avg_Pct, PTU_Max_Pct, Ratelimit, Spillover, Aggregated_PTU_With_Spillover, PTU_Normalizado\n| order by TimeGenerated asc", - "size": 0, - "title": "PTU utilization and PAYG spillover", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "capacity-ptu" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureMetrics\n| where MetricName in ('AzureOpenAIProvisionedManagedUtilizationV2','Ratelimit','TokenTransaction','ProcessedPromptTokens')\n| extend Deployment = tostring(split(Resource, '/')[-1])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| summarize PTU_Avg_Pct = avgif(Average, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n PTU_Max_Pct = maxif(Maximum, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n Ratelimit = avgif(Average, MetricName == 'Ratelimit'),\n ProcessedTokens = sumif(Total, MetricName in ('TokenTransaction','ProcessedPromptTokens'))\n by TimeGenerated = bin(TimeGenerated, 1m), Deployment\n| extend Spillover = max_of(0.0, ProcessedTokens - Ratelimit)\n| extend Aggregated_PTU_With_Spillover = ProcessedTokens\n| extend PTU_Normalizado = PTU_Avg_Pct * 100000\n| project TimeGenerated, PTU_Avg_Pct, PTU_Max_Pct, Ratelimit, Spillover, Aggregated_PTU_With_Spillover, PTU_Normalizado\n| order by TimeGenerated asc", - "size": 0, - "title": "Capacity summary (per bin)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "capacity-ptu-table" - } - ] - }, - "name": "group-capacity" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "traffic" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "### Traffic and tokens\nCall volume and token consumption broken down by deployment and model. The PTU vs PAYG pivot splits token volume by capacity type (streaming traffic maps to PTU, non-streaming overflow to PAYG spillover)." - }, - "name": "traffic-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), Model = tostring(props.modelName), StreamType = tostring(props.streamType),\n PromptTokens = toreal(props.promptTokens[0]), GeneratedTokens = toreal(props.generatedTokens[0])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{Model}' == 'value::all' or Model == '{Model}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize AzureOpenAIRequests = count(), InputTokens = sum(PromptTokens), OutputTokens = sum(GeneratedTokens),\n ProcessedPromptTokens = sum(PromptTokens) by TimeGenerated = bin(TimeGenerated, 5m), Deployment, Model\n| extend TotalTokens = InputTokens + OutputTokens, GeneratedTokens = OutputTokens, TotalCalls = AzureOpenAIRequests\n| extend TokensPorRequest = iff(AzureOpenAIRequests > 0, ProcessedPromptTokens / AzureOpenAIRequests, 0.0)\n| order by TimeGenerated asc", - "size": 0, - "title": "Calls and tokens over time", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "traffic-tokens" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), Model = tostring(props.modelName), StreamType = tostring(props.streamType),\n PromptTokens = toreal(props.promptTokens[0]), GeneratedTokens = toreal(props.generatedTokens[0])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{Model}' == 'value::all' or Model == '{Model}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize AzureOpenAIRequests = count(), InputTokens = sum(PromptTokens), OutputTokens = sum(GeneratedTokens),\n ProcessedPromptTokens = sum(PromptTokens) by TimeGenerated = bin(TimeGenerated, 5m), Deployment, Model\n| extend TotalTokens = InputTokens + OutputTokens, GeneratedTokens = OutputTokens, TotalCalls = AzureOpenAIRequests\n| extend TokensPorRequest = iff(AzureOpenAIRequests > 0, ProcessedPromptTokens / AzureOpenAIRequests, 0.0)\n| order by TimeGenerated asc", - "size": 0, - "title": "Tokens per request (by deployment and model)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "traffic-tokens-table" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType),\n PromptTokens = toreal(props.promptTokens[0]), GeneratedTokens = toreal(props.generatedTokens[0])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| extend CapacityType = iff(StreamType == 'Streaming', 'PTU', 'PAYGO_SPILLOVER')\n| summarize Tokens = sum(PromptTokens + GeneratedTokens) by TimeGenerated = bin(TimeGenerated, 5m), CapacityType\n| evaluate pivot(CapacityType, sum(Tokens))\n| order by TimeGenerated asc", - "size": 0, - "title": "PTU vs PAYG spillover (token volume)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "traffic-pivot" - } - ] - }, - "name": "group-traffic" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "latency" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "### Latency\nNormalized Azure OpenAI latency metrics (TTFT, TBT, TTLT, tokens/sec, time-to-response) plus P95/P99/Avg/Max computed from per-request `DurationMs`." - }, - "name": "latency-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureMetrics\n| where MetricName in ('AzureOpenAINormalizedTTFTInMS','AzureOpenAINormalizedTBTInMS','AzureOpenAITTLTInMS','AzureOpenAITokenPerSecond','AzureOpenAITimeToResponse','Latency')\n| extend Deployment = tostring(split(Resource, '/')[-1])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| summarize TTFT_Ms = avgif(Average, MetricName == 'AzureOpenAINormalizedTTFTInMS'),\n TBT_Ms = avgif(Average, MetricName == 'AzureOpenAINormalizedTBTInMS'),\n TTLT_Ms = avgif(Average, MetricName == 'AzureOpenAITTLTInMS'),\n TokensPerSecond = avgif(Average, MetricName == 'AzureOpenAITokenPerSecond'),\n TimeToResponse_Ms = avgif(Average, MetricName == 'AzureOpenAITimeToResponse'),\n Latency_Ms = avgif(Average, MetricName == 'Latency') by TimeGenerated = bin(TimeGenerated, 5m)\n| order by TimeGenerated asc", - "size": 0, - "title": "Normalized latency and throughput", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "latency-platform" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| where isnotnull(DurationMs)\n| summarize P95LatencyMs = percentile(DurationMs, 95), P99LatencyMs = percentile(DurationMs, 99),\n AvgLatencyMs = avg(DurationMs), MaxLatencyMs = max(DurationMs) by TimeGenerated = bin(TimeGenerated, 5m)\n| order by TimeGenerated asc", - "size": 0, - "title": "DurationMs percentiles (P95 / P99 / Avg / Max)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "latency-percentiles" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| where isnotnull(DurationMs)\n| summarize P95LatencyMs = percentile(DurationMs, 95), P99LatencyMs = percentile(DurationMs, 99),\n AvgLatencyMs = avg(DurationMs), MaxLatencyMs = max(DurationMs) by TimeGenerated = bin(TimeGenerated, 5m)\n| order by TimeGenerated asc", - "size": 0, - "title": "Latency percentiles (per bin)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "latency-percentiles-table" - } - ] - }, - "name": "group-latency" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "errors" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "### Errors and throttling\nHTTP status breakdown (429 / 400 / 500), blocked calls, and derived throttling / error rates." - }, - "name": "errors-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType), Status = toint(ResultSignature)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize TotalRequests = count(), HTTP429 = countif(Status == 429), HTTP400 = countif(Status == 400),\n HTTP500 = countif(Status >= 500 and Status < 600), Successful = countif(Status == 200)\n by TimeGenerated = bin(TimeGenerated, 5m)\n| extend BlockedCalls = HTTP429\n| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0)\n| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0)\n| project TimeGenerated, BlockedCalls, HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct\n| order by TimeGenerated asc", - "size": 0, - "title": "HTTP status counts over time", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "errors-counts" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType), Status = toint(ResultSignature)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize TotalRequests = count(), HTTP429 = countif(Status == 429), HTTP400 = countif(Status == 400),\n HTTP500 = countif(Status >= 500 and Status < 600), Successful = countif(Status == 200)\n by TimeGenerated = bin(TimeGenerated, 5m)\n| extend BlockedCalls = HTTP429\n| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0)\n| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0)\n| project TimeGenerated, BlockedCalls, HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct\n| order by TimeGenerated asc", - "size": 0, - "title": "Throttling rate and error rate (%)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "errors-rates" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType), Status = toint(ResultSignature)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize TotalRequests = count(), HTTP429 = countif(Status == 429), HTTP400 = countif(Status == 400),\n HTTP500 = countif(Status >= 500 and Status < 600), Successful = countif(Status == 200)\n by TimeGenerated = bin(TimeGenerated, 5m)\n| extend BlockedCalls = HTTP429\n| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0)\n| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0)\n| project TimeGenerated, BlockedCalls, HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct\n| order by TimeGenerated asc", - "size": 0, - "title": "Errors and throttling (per bin)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "errors-table" - } - ] - }, - "name": "group-errors" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "agent-behavior" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "## Agent behavior\n> **Preview - Foundry platform-owned evaluation signals.** Official Foundry [trace annotation documentation](https://learn.microsoft.com/azure/foundry/observability/how-to/trace-annotations#log-end-user-feedback-as-trace-annotations) verifies `gen_ai.evaluation.result` events for **human trace annotations**. Automated trace-evaluation results are shown only when the target workspace proves that it exports the same compatible event shape; that producer support is validation-dependent. AgentOps reads compatible events from Azure Monitor and does not create, schedule, gate, or edit evaluations.\n\nThe status and freshness row appears before quality results. **Observed invoke_agent invocations**, **evaluated traces**, and **evaluation events** are separate counts because one trace can produce several evaluator events and this workspace may not contain a complete invocation denominator. Trace-ID correlation does not require `gen_ai.agent.id`; missing agent/version metadata remains visible as not reported.\n\n**State guide:** `Schema unavailable` means matching events exist but the bounded `agent_behavior/v1` property mapping is not recognized. `No access` appears as the native workbook query permission error; request **Log Analytics Reader** on the workspace. `No data` means no compatible events exist in the selected time range; it can also mean automated trace evaluation does not export this event shape in the target environment. `Filter empty` means events exist but none match these filters. `Possible ingestion delay` means the newest matching event is more than 15 minutes old; it can also mean no recent compatible evaluation signal was produced." - }, - "name": "agent-behavior-note" - }, - { - "type": 9, - "content": { - "version": "KqlParameterItem/1.0", - "parameters": [ - { - "id": "p-agent-environment", - "version": "KqlParameterItem/1.0", - "name": "AgentEnvironment", - "label": "Environment (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - }, - { - "id": "p-agent-name", - "version": "KqlParameterItem/1.0", - "name": "AgentName", - "label": "Agent (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - }, - { - "id": "p-agent-version", - "version": "KqlParameterItem/1.0", - "name": "AgentVersion", - "label": "Version (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - }, - { - "id": "p-evaluator", - "version": "KqlParameterItem/1.0", - "name": "Evaluator", - "label": "Evaluator (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - } - ], - "style": "pills", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces" - }, - "name": "agent-behavior-filters" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nlet SchemaVersion = 'agent_behavior/v1';\nlet EvaluationEventsV1 = materialize(union isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, SourceTable='AppEvents', MonitorTraceId=tostring(column_ifexists('OperationId','')), RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, SourceTable='customEvents', MonitorTraceId=tostring(column_ifexists('operation_Id','')), RawProperties=todynamic(customDimensions))\n| extend EvaluatorRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), LabelRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), AgentRaw=tostring(RawProperties['gen_ai.agent.name']), VersionRaw=tostring(RawProperties['gen_ai.agent.version']), EnvironmentRaw=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment'])), PropertyTraceId=coalesce(tostring(RawProperties['trace_id']),tostring(RawProperties['gen_ai.trace.id']))\n| extend Agent=coalesce(AgentRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(VersionRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Environment=EnvironmentRaw, TraceId=coalesce(PropertyTraceId,MonitorTraceId), SchemaRecognized=isnotempty(EvaluatorRaw) or isnotempty(ScoreText) or isnotempty(LabelRaw)\n| extend Evaluator=iff(isempty(EvaluatorRaw),'Evaluator not reported',EvaluatorRaw), Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), TraceId=iff(isempty(TraceId),'Trace ID not reported',TraceId)\n| project EventTime,SourceTable,TraceId,Agent,Version,Environment,Evaluator,SchemaRecognized);\nlet InvokeAgentV1 = materialize(union isfuzzy=true\n (AppDependencies | project InvocationTime=TimeGenerated, SourceTable='AppDependencies', EventId=tostring(column_ifexists('Id','')), MonitorTraceId=tostring(column_ifexists('OperationId','')), RawProperties=todynamic(Properties)),\n (dependencies | project InvocationTime=timestamp, SourceTable='dependencies', EventId=tostring(column_ifexists('id','')), MonitorTraceId=tostring(column_ifexists('operation_Id','')), RawProperties=todynamic(customDimensions))\n| where tostring(RawProperties['gen_ai.operation.name']) == 'invoke_agent'\n| extend AgentId=tostring(RawProperties['gen_ai.agent.id']), AgentRaw=tostring(RawProperties['gen_ai.agent.name']), VersionRaw=tostring(RawProperties['gen_ai.agent.version']), EnvironmentRaw=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment'])), TraceId=coalesce(tostring(RawProperties['trace_id']),tostring(RawProperties['gen_ai.trace.id']),MonitorTraceId)\n| extend Agent=coalesce(AgentRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(VersionRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Environment=EnvironmentRaw\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), InvocationKey=coalesce(TraceId,EventId,strcat(SourceTable,'|',format_datetime(InvocationTime,'o')))\n| project InvocationTime,InvocationKey,Agent,Version,Environment);\nlet FilteredEvents = EvaluationEventsV1 | where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}';\nlet FilteredInvocations = InvokeAgentV1 | where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}';\nlet AllSummary = EvaluationEventsV1 | summarize RawEvaluationEvents=count(), RecognizedEvaluationEvents=countif(SchemaRecognized), LatestAvailableEvent=max(EventTime);\nlet FilteredSummary = FilteredEvents | summarize EvaluationEvents=count(), EvaluatedTraces=dcountif(TraceId,TraceId != 'Trace ID not reported'), LatestMatchingEvent=max(EventTime);\nlet InvocationSummary = FilteredInvocations | summarize ObservedInvokeAgentInvocations=dcount(InvocationKey);\nAllSummary | extend JoinKey=1 | join kind=inner (FilteredSummary | extend JoinKey=1) on JoinKey | join kind=inner (InvocationSummary | extend JoinKey=1) on JoinKey\n| extend FreshnessMinutes=datetime_diff('minute',now(),LatestMatchingEvent)\n| extend DataStatus=case(RawEvaluationEvents == 0,'No data',RecognizedEvaluationEvents == 0,'Schema unavailable',EvaluationEvents == 0,'Filter empty',FreshnessMinutes > 15,'Possible ingestion delay','Ready')\n | project NormalizationVersion=SchemaVersion,ProducerSupport='Human annotations documented; automated trace-evaluation export validation-dependent',DataStatus,FreshnessMinutes,LatestMatchingEvent,ObservedInvokeAgentInvocations,EvaluatedTraces,EvaluationEvents,RawEvaluationEvents,RecognizedEvaluationEvents\n", - "size": 0, - "title": "Data status, freshness, and separate observed counts", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-status" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, SourceTable='AppEvents', RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, SourceTable='customEvents', RawProperties=todynamic(customDimensions))\n| extend EvaluatorRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), LabelRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))\n| extend SchemaRecognized=isnotempty(EvaluatorRaw) or isnotempty(ScoreText) or isnotempty(LabelRaw), MissingEvaluator=isempty(EvaluatorRaw), MissingScore=isempty(ScoreText), MissingLabel=isempty(LabelRaw)\n| where not(SchemaRecognized) or MissingEvaluator or MissingScore or MissingLabel\n| project EventTime,SourceTable,SchemaRecognized,MissingEvaluator,MissingScore,MissingLabel,RawProperties\n| order by EventTime desc\n| take 100", - "size": 0, - "title": "Schema diagnostics (raw properties retained)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-schema-diagnostics" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nlet EvaluationEventsV1 = materialize(union isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend EvaluatorRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), Label=tolower(coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(EvaluatorRaw),'Evaluator not reported',EvaluatorRaw), Score=todouble(ScoreText)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), IsPass=Label in ('pass','passed','true','relevant','correct'), IsFail=Label in ('fail','failed','false','not_relevant','incorrect')\n| project EventTime,Agent,Version,Environment,Evaluator,Score,ScoreText,IsPass,IsFail);\nEvaluationEventsV1 | where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| summarize EvaluationEvents=count(), LabeledEvents=countif(IsPass or IsFail), PassingEvents=countif(IsPass), FailingEvents=countif(IsFail), NumericScores=countif(isnotnull(Score)), AverageScore=avg(Score) by Evaluator\n| extend PassRatePct=iff(LabeledEvents > 0,round(100.0 * PassingEvents / LabeledEvents,1),real(null))\n| project Evaluator,EvaluationEvents,LabeledEvents,PassRatePct,PassingEvents,FailingEvents,NumericScores,AverageScore\n| order by EvaluationEvents desc", - "size": 0, - "title": "Per-evaluator pass rate, volume, and raw-score summary", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-evaluator-summary" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), Label=tolower(coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), IsPass=Label in ('pass','passed','true','relevant','correct'), IsFail=Label in ('fail','failed','false','not_relevant','incorrect')\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| summarize LabeledEvents=countif(IsPass or IsFail), PassingEvents=countif(IsPass) by EventTime=bin(EventTime,1h),Evaluator\n| where LabeledEvents > 0\n| extend PassRatePct=round(100.0 * PassingEvents / LabeledEvents,1)\n| project EventTime,Evaluator,PassRatePct\n| order by EventTime asc", - "size": 0, - "title": "Pass-rate trend by evaluator (%)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "agent-behavior-pass-trend" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment)\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| summarize EvaluationEvents=count() by EventTime=bin(EventTime,1h),Evaluator\n| project EventTime,Evaluator,EvaluationEvents\n| order by EventTime asc", - "size": 0, - "title": "Evaluation-event volume trend by evaluator", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "agent-behavior-volume-trend" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator), Score=todouble(ScoreText)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment)\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| where isnotnull(Score)\n| summarize AverageScore=avg(Score),MinimumScore=min(Score),MaximumScore=max(Score),NumericScoreEvents=count() by EventTime=bin(EventTime,1h),Evaluator\n| project EventTime,Evaluator,AverageScore,MinimumScore,MaximumScore,NumericScoreEvents\n| order by EventTime desc,Evaluator asc", - "size": 0, - "title": "Raw-score trend by evaluator (separate scales; do not compare evaluators)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-score-trend" - }, - { - "type": 1, - "content": { - "json": "### Recent failed or lowest-scored evaluation events\nRaw numeric scores are ordered only within the event list and are **not comparable across unlike evaluators**. Copy a trace ID, open the matching project in **Microsoft Foundry > Tracing**, and search for that trace ID. A reliable Foundry project deep link cannot be constructed from the workbook workspace context alone." - }, - "name": "agent-behavior-trace-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, SourceTable='AppEvents', MonitorTraceId=tostring(column_ifexists('OperationId','')), RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, SourceTable='customEvents', MonitorTraceId=tostring(column_ifexists('operation_Id','')), RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), Label=tolower(coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))), Explanation=coalesce(tostring(RawProperties['gen_ai.evaluation.explanation']),tostring(RawProperties['gen_ai.evaluation.reason'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment'])), TraceId=coalesce(tostring(RawProperties['trace_id']),tostring(RawProperties['gen_ai.trace.id']),MonitorTraceId), ResponseId=tostring(RawProperties['gen_ai.response.id']), ConversationId=tostring(RawProperties['gen_ai.conversation.id'])\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator), Score=todouble(ScoreText)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), TraceId=iff(isempty(TraceId),'Trace ID not reported',TraceId), IsFail=Label in ('fail','failed','false','not_relevant','incorrect')\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| extend ReviewPriority=case(IsFail,0,isnotnull(Score),1,2)\n| top 100 by ReviewPriority asc,Score asc,EventTime desc\n| project EventTime,Evaluator,Score,ScoreText,Label,Agent,Version,Environment,TraceId,ResponseId,ConversationId,Explanation,SourceTable\n", - "size": 0, - "title": "Recent failed or lowest-scored events with correlation IDs", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-recent-traces" - } - ] - }, - "name": "group-agent-behavior" - } - ], - "isLocked": false, - "fallbackResourceIds": [ - "Azure Monitor" - ], - "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" -} diff --git a/src/agentops/templates/workbooks/queries/agent_behavior.kql b/src/agentops/templates/workbooks/queries/agent_behavior.kql deleted file mode 100644 index 1d0dbe00..00000000 --- a/src/agentops/templates/workbooks/queries/agent_behavior.kql +++ /dev/null @@ -1,222 +0,0 @@ -// Agent behavior normalization contract: agent_behavior/v1 -// -// Official Foundry docs verify gen_ai.evaluation.result events for human trace -// annotations. Automated trace-evaluation export through this same event shape -// is validation-dependent and must be proven in the target workspace. AgentOps -// reads compatible events and observed invoke_agent spans without creating, -// scheduling, or changing evaluations. The bounded query keeps an explicit -// time window and projects a stable v1 shape while preserving each raw bag. -let _startTime = ago(24h); -let _endTime = now(); -let _environment = "*"; -let _agent = "*"; -let _version = "*"; -let _evaluator = "*"; -let _normalizationVersion = "agent_behavior/v1"; -set best_effort=true; -let EvaluationEventsV1 = materialize( - union isfuzzy=true - ( - AppEvents - | where TimeGenerated between (_startTime .. _endTime) - | where Name == "gen_ai.evaluation.result" - | project - EventTime = TimeGenerated, - SourceTable = "AppEvents", - EventId = tostring(column_ifexists("Id", "")), - MonitorTraceId = tostring(column_ifexists("OperationId", "")), - RawProperties = todynamic(Properties) - ), - ( - customEvents - | where timestamp between (_startTime .. _endTime) - | where name == "gen_ai.evaluation.result" - | project - EventTime = timestamp, - SourceTable = "customEvents", - EventId = tostring(column_ifexists("id", "")), - MonitorTraceId = tostring(column_ifexists("operation_Id", "")), - RawProperties = todynamic(customDimensions) - ) - | extend - EvaluatorRaw = coalesce( - tostring(RawProperties["gen_ai.evaluation.name"]), - tostring(RawProperties["evaluator"]) - ), - ScoreText = coalesce( - tostring(RawProperties["gen_ai.evaluation.score.value"]), - tostring(RawProperties["gen_ai.evaluation.score"]), - tostring(RawProperties["score"]) - ), - LabelRaw = coalesce( - tostring(RawProperties["gen_ai.evaluation.score.label"]), - tostring(RawProperties["gen_ai.evaluation.result"]), - tostring(RawProperties["label"]) - ), - Explanation = coalesce( - tostring(RawProperties["gen_ai.evaluation.explanation"]), - tostring(RawProperties["gen_ai.evaluation.reason"]) - ), - AgentId = tostring(RawProperties["gen_ai.agent.id"]), - AgentRaw = tostring(RawProperties["gen_ai.agent.name"]), - VersionRaw = tostring(RawProperties["gen_ai.agent.version"]), - EnvironmentRaw = coalesce( - tostring(RawProperties["deployment.environment.name"]), - tostring(RawProperties["gen_ai.environment"]), - tostring(RawProperties["environment"]) - ), - PropertyTraceId = coalesce( - tostring(RawProperties["trace_id"]), - tostring(RawProperties["gen_ai.trace.id"]) - ), - ResponseId = tostring(RawProperties["gen_ai.response.id"]), - ConversationId = tostring(RawProperties["gen_ai.conversation.id"]) - | extend - Score = todouble(ScoreText), - Label = tolower(LabelRaw), - Agent = coalesce( - AgentRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[0]), AgentId) - ), - Version = coalesce( - VersionRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[1]), "") - ), - Environment = EnvironmentRaw, - TraceId = coalesce(PropertyTraceId, MonitorTraceId), - SchemaRecognized = isnotempty(EvaluatorRaw) - or isnotempty(ScoreText) - or isnotempty(LabelRaw) - | extend - Evaluator = iff(isempty(EvaluatorRaw), "Evaluator not reported", EvaluatorRaw), - Agent = iff(isempty(Agent), "Agent not reported", Agent), - Version = iff(isempty(Version), "Version not reported", Version), - Environment = iff( - isempty(Environment), - "Environment not reported", - Environment - ), - TraceId = iff(isempty(TraceId), "Trace ID not reported", TraceId), - ScoreIsNumeric = isnotempty(ScoreText) and isnotnull(Score), - IsPass = Label in ("pass", "passed", "true", "relevant", "correct"), - IsFail = Label in ( - "fail", - "failed", - "false", - "not_relevant", - "incorrect" - ), - NormalizationVersion = _normalizationVersion - | project - EventTime, - SourceTable, - EventId, - TraceId, - ResponseId, - ConversationId, - Agent, - Version, - Environment, - Evaluator, - Score, - ScoreText, - ScoreIsNumeric, - Label, - Explanation, - IsPass, - IsFail, - SchemaRecognized, - NormalizationVersion, - RawProperties -); -let InvokeAgentInvocationsV1 = materialize( - union isfuzzy=true - ( - AppDependencies - | where TimeGenerated between (_startTime .. _endTime) - | project - InvocationTime = TimeGenerated, - SourceTable = "AppDependencies", - EventId = tostring(column_ifexists("Id", "")), - MonitorTraceId = tostring(column_ifexists("OperationId", "")), - RawProperties = todynamic(Properties) - ), - ( - dependencies - | where timestamp between (_startTime .. _endTime) - | project - InvocationTime = timestamp, - SourceTable = "dependencies", - EventId = tostring(column_ifexists("id", "")), - MonitorTraceId = tostring(column_ifexists("operation_Id", "")), - RawProperties = todynamic(customDimensions) - ) - | where tostring(RawProperties["gen_ai.operation.name"]) == "invoke_agent" - | extend - AgentId = tostring(RawProperties["gen_ai.agent.id"]), - AgentRaw = tostring(RawProperties["gen_ai.agent.name"]), - VersionRaw = tostring(RawProperties["gen_ai.agent.version"]), - EnvironmentRaw = coalesce( - tostring(RawProperties["deployment.environment.name"]), - tostring(RawProperties["gen_ai.environment"]), - tostring(RawProperties["environment"]) - ), - PropertyTraceId = coalesce( - tostring(RawProperties["trace_id"]), - tostring(RawProperties["gen_ai.trace.id"]) - ) - | extend - Agent = coalesce( - AgentRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[0]), AgentId) - ), - Version = coalesce( - VersionRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[1]), "") - ), - Environment = EnvironmentRaw, - TraceId = coalesce(PropertyTraceId, MonitorTraceId) - | extend - Agent = iff(isempty(Agent), "Agent not reported", Agent), - Version = iff(isempty(Version), "Version not reported", Version), - Environment = iff( - isempty(Environment), - "Environment not reported", - Environment - ), - InvocationKey = coalesce( - TraceId, - EventId, - strcat(SourceTable, "|", format_datetime(InvocationTime, "o")) - ), - NormalizationVersion = _normalizationVersion - | project - InvocationTime, - InvocationKey, - TraceId, - Agent, - Version, - Environment, - SourceTable, - NormalizationVersion, - RawProperties -); -EvaluationEventsV1 -| where _environment == "*" or Environment == _environment -| where _agent == "*" or Agent == _agent -| where _version == "*" or Version == _version -| where _evaluator == "*" or Evaluator == _evaluator -| summarize - EvaluationEvents = count(), - EvaluatedTraces = dcountif(TraceId, TraceId != "Trace ID not reported"), - NumericScores = countif(ScoreIsNumeric), - PassLabels = countif(IsPass), - FailLabels = countif(IsFail), - PassRatePct = round( - 100.0 * countif(IsPass) / max_of(1, countif(IsPass or IsFail)), - 1 - ), - AverageScore = avg(Score), - LatestEvent = max(EventTime) - by Evaluator, Version, NormalizationVersion -| order by EvaluationEvents desc diff --git a/src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql b/src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql deleted file mode 100644 index 5f043fbf..00000000 --- a/src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql +++ /dev/null @@ -1,52 +0,0 @@ -// Capacity: PTU utilization, rate limit, and PAYG spillover -// ----------------------------------------------------------------------------- -// Source: AzureMetrics (platform metrics emitted by the Azure OpenAI resource). -// Derived metrics: PTU_Avg_Pct, PTU_Max_Pct, Ratelimit, Spillover volume, -// Aggregated_PTU_With_Spillover, PTU_Normalizado. -// -// The foundry-ops workbook binds the following parameters into this query at -// runtime. When running the query standalone in Log Analytics, edit the `let` -// bindings below to match your environment. -// {TimeRange} -> replaced by the workbook time-range picker (KQL timespan) -// {Resource} -> Azure OpenAI resource id (used as the workbook scope) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// -// Note on PTU_Normalizado: this is a DISPLAY-ONLY series (PTU% * 100000). It -// lets the 0-100 PTU utilization percentage share a Y axis with raw token -// counts in a single timechart. Do not treat it as a real token volume. -// ----------------------------------------------------------------------------- -let _bin = 1m; -let _deployment = "*"; // workbook: {Deployment} -AzureMetrics -| where TimeGenerated {TimeRange} -| where MetricName in ( - "AzureOpenAIProvisionedManagedUtilizationV2", - "Ratelimit", - "TokenTransaction", - "ProcessedPromptTokens" - ) -| extend Deployment = tostring(split(Resource, "/")[-1]) -| where _deployment == "*" or Deployment == _deployment -| summarize - PTU_Avg_Pct = avgif(Average, MetricName == "AzureOpenAIProvisionedManagedUtilizationV2"), - PTU_Max_Pct = maxif(Maximum, MetricName == "AzureOpenAIProvisionedManagedUtilizationV2"), - Ratelimit = avgif(Average, MetricName == "Ratelimit"), - ProcessedTokens = sumif(Total, MetricName in ("TokenTransaction", "ProcessedPromptTokens")) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment -// Tokens above the deployment rate limit spill over to pay-as-you-go capacity. -| extend Spillover = max_of(0.0, ProcessedTokens - Ratelimit) -| extend PTU_Tokens = ProcessedTokens - Spillover -| extend Aggregated_PTU_With_Spillover = PTU_Tokens + Spillover -// Display-only scaling series so PTU % can share a Y axis with token counts. -| extend PTU_Normalizado = PTU_Avg_Pct * 100000 -| project - TimeGenerated, - Deployment, - PTU_Avg_Pct, - PTU_Max_Pct, - Ratelimit, - Spillover, - Aggregated_PTU_With_Spillover, - PTU_Normalizado -| order by TimeGenerated asc diff --git a/src/agentops/templates/workbooks/queries/errors_throttling.kql b/src/agentops/templates/workbooks/queries/errors_throttling.kql deleted file mode 100644 index 6099381e..00000000 --- a/src/agentops/templates/workbooks/queries/errors_throttling.kql +++ /dev/null @@ -1,48 +0,0 @@ -// Errors and throttling: HTTP status breakdown, throttling rate, error rate -// ----------------------------------------------------------------------------- -// Source: AzureDiagnostics (ResultSignature holds the HTTP status) and -// AzureMetrics (BlockedCalls / TotalCalls) for platform-side throttling counts. -// Derived metrics: HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct, -// BlockedCalls. -// -// Workbook parameter bindings (edit the `let` lines to run standalone): -// {TimeRange} -> workbook time-range picker (KQL timespan) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// ----------------------------------------------------------------------------- -let _bin = 5m; -let _deployment = "*"; // workbook: {Deployment} -let _stream = "*"; // workbook: {StreamType} -AzureDiagnostics -| where TimeGenerated {TimeRange} -| where OperationName in ("ChatCompletions_Create", "RequestResponse") -| extend props = parse_json(properties_s) -| extend - Deployment = tostring(props.modelDeploymentName), - StreamType = tostring(props.streamType), - Status = toint(ResultSignature) -| where _deployment == "*" or Deployment == _deployment -| where _stream == "*" or StreamType == _stream -| summarize - TotalRequests = count(), - HTTP429 = countif(Status == 429), - HTTP400 = countif(Status == 400), - HTTP500 = countif(Status >= 500 and Status < 600), - Successful = countif(Status == 200) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment -| extend BlockedCalls = HTTP429 -// Throttling rate: share of requests rejected with HTTP 429. -| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0) -// Error rate: share of requests that did not return HTTP 200. -| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0) -| project - TimeGenerated, - Deployment, - TotalRequests, - BlockedCalls, - HTTP429, - HTTP400, - HTTP500, - TasaThrottling_Pct, - TasaError_Pct -| order by TimeGenerated asc diff --git a/src/agentops/templates/workbooks/queries/latency_percentiles.kql b/src/agentops/templates/workbooks/queries/latency_percentiles.kql deleted file mode 100644 index 5ceb5ce6..00000000 --- a/src/agentops/templates/workbooks/queries/latency_percentiles.kql +++ /dev/null @@ -1,74 +0,0 @@ -// Latency: TTFT, TBT, TTLT, tokens/sec, and DurationMs percentiles -// ----------------------------------------------------------------------------- -// Source: AzureMetrics for the normalized Azure OpenAI latency metrics, and -// AzureDiagnostics (DurationMs) for per-request percentiles. -// Derived metrics: P95LatencyMs, P99LatencyMs, AvgLatencyMs, MaxLatencyMs, plus -// the platform latency metrics surfaced side by side. -// -// Workbook parameter bindings (edit the `let` lines to run standalone): -// {TimeRange} -> workbook time-range picker (KQL timespan) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// ----------------------------------------------------------------------------- -let _bin = 5m; -let _deployment = "*"; // workbook: {Deployment} -let _stream = "*"; // workbook: {StreamType} -// Platform-normalized latency and throughput metrics. -let platform = - AzureMetrics - | where TimeGenerated {TimeRange} - | where MetricName in ( - "AzureOpenAINormalizedTTFTInMS", - "AzureOpenAINormalizedTBTInMS", - "AzureOpenAITTLTInMS", - "AzureOpenAITokenPerSecond", - "AzureOpenAITimeToResponse", - "Latency" - ) - | extend Deployment = tostring(split(Resource, "/")[-1]) - | where _deployment == "*" or Deployment == _deployment - | summarize - TTFT_Ms = avgif(Average, MetricName == "AzureOpenAINormalizedTTFTInMS"), - TBT_Ms = avgif(Average, MetricName == "AzureOpenAINormalizedTBTInMS"), - TTLT_Ms = avgif(Average, MetricName == "AzureOpenAITTLTInMS"), - TokensPerSecond = avgif(Average, MetricName == "AzureOpenAITokenPerSecond"), - TimeToResponse_Ms = avgif(Average, MetricName == "AzureOpenAITimeToResponse"), - Latency_Ms = avgif(Average, MetricName == "Latency") - by TimeGenerated = bin(TimeGenerated, _bin), Deployment; -// Per-request duration percentiles from diagnostic records. -let durations = - AzureDiagnostics - | where TimeGenerated {TimeRange} - | where OperationName in ("ChatCompletions_Create", "RequestResponse") - | extend props = parse_json(properties_s) - | extend - Deployment = tostring(props.modelDeploymentName), - StreamType = tostring(props.streamType) - | where _deployment == "*" or Deployment == _deployment - | where _stream == "*" or StreamType == _stream - | where isnotnull(DurationMs) - | summarize - P95LatencyMs = percentile(DurationMs, 95), - P99LatencyMs = percentile(DurationMs, 99), - AvgLatencyMs = avg(DurationMs), - MaxLatencyMs = max(DurationMs) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment; -platform -| join kind=fullouter durations on TimeGenerated, Deployment -| extend - TimeGenerated = coalesce(TimeGenerated, TimeGenerated1), - Deployment = coalesce(Deployment, Deployment1) -| project - TimeGenerated, - Deployment, - TTFT_Ms, - TBT_Ms, - TTLT_Ms, - TokensPerSecond, - TimeToResponse_Ms, - Latency_Ms, - P95LatencyMs, - P99LatencyMs, - AvgLatencyMs, - MaxLatencyMs -| order by TimeGenerated asc diff --git a/src/agentops/templates/workbooks/queries/traffic_tokens.kql b/src/agentops/templates/workbooks/queries/traffic_tokens.kql deleted file mode 100644 index b8946cff..00000000 --- a/src/agentops/templates/workbooks/queries/traffic_tokens.kql +++ /dev/null @@ -1,61 +0,0 @@ -// Traffic and tokens: call volume, token consumption, and PTU vs PAYG split -// ----------------------------------------------------------------------------- -// Source: AzureDiagnostics (Category == "AzureOpenAIRequestUsage") for real -// per-model token usage, joined with AzureMetrics for platform call volume. -// Derived metrics: TokensPorRequest, InputTokens, OutputTokens, TotalTokens, -// ProcessedPromptTokens, GeneratedTokens, and a PTU vs PAYG -// spillover pivot. -// -// Workbook parameter bindings (edit the `let` lines to run standalone): -// {TimeRange} -> workbook time-range picker (KQL timespan) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {Model} -> modelName filter, or "*" for all models -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// ----------------------------------------------------------------------------- -let _bin = 5m; -let _deployment = "*"; // workbook: {Deployment} -let _model = "*"; // workbook: {Model} -let _stream = "*"; // workbook: {StreamType} -AzureDiagnostics -| where TimeGenerated {TimeRange} -| where Category == "AzureOpenAIRequestUsage" -| extend props = parse_json(properties_s) -| extend - Deployment = tostring(props.modelDeploymentName), - Model = tostring(props.modelName), - ModelVersion = tostring(props.modelVersion), - StreamType = tostring(props.streamType), - PromptTokens = toreal(props.promptTokens[0]), - GeneratedTokens = toreal(props.generatedTokens[0]) -| where _deployment == "*" or Deployment == _deployment -| where _model == "*" or Model == _model -| where _stream == "*" or StreamType == _stream -| summarize - AzureOpenAIRequests = count(), - InputTokens = sum(PromptTokens), - OutputTokens = sum(GeneratedTokens), - ProcessedPromptTokens = sum(PromptTokens) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment, Model, StreamType -| extend TotalTokens = InputTokens + OutputTokens -| extend GeneratedTokens = OutputTokens -| extend TotalCalls = AzureOpenAIRequests -// Average tokens billed per request over the bin. -| extend TokensPorRequest = iff(AzureOpenAIRequests > 0, ProcessedPromptTokens / AzureOpenAIRequests, 0.0) -// PTU vs PAYG split: streaming and provisioned traffic run on PTU capacity; -// non-streaming overflow is treated as the PAYG spillover pivot column. -| extend CapacityType = iff(StreamType == "Streaming", "PTU", "PAYGO_SPILLOVER") -| project - TimeGenerated, - Deployment, - Model, - StreamType, - CapacityType, - TotalCalls, - AzureOpenAIRequests, - InputTokens, - OutputTokens, - TotalTokens, - ProcessedPromptTokens, - GeneratedTokens, - TokensPorRequest -| order by TimeGenerated asc diff --git a/src/agentops/utils/foundry_discovery.py b/src/agentops/utils/foundry_discovery.py index 8737b3a2..50fb1604 100644 --- a/src/agentops/utils/foundry_discovery.py +++ b/src/agentops/utils/foundry_discovery.py @@ -283,6 +283,81 @@ def resolve_appinsights_connection(project_endpoint: str) -> Optional[str]: return conn +def resolve_appinsights_resource_id_with_reason( + project_endpoint: str, +) -> Tuple[Optional[str], Optional[str]]: + """Return the ARM resource ID of the Foundry-linked App Insights resource. + + Connection metadata is credential-free, so this works for both API-key and + ProjectManagedIdentity connections without requesting connection secrets. + """ + if not project_endpoint: + return None, "no AZURE_AI_FOUNDRY_PROJECT_ENDPOINT set" + + cache_key = f"appinsights-resource:{project_endpoint}" + cached = _lookup(cache_key) + if cached is not None: + return cached + + try: + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + except ImportError: + reason = ( + "azure-ai-projects / azure-identity not installed in the cockpit's " + "Python environment. Install with " + "`pip install azure-ai-projects azure-identity`." + ) + _store(cache_key, None, reason) + return None, reason + + try: + credential = DefaultAzureCredential( + exclude_developer_cli_credential=True, + process_timeout=30, + ) + client = AIProjectClient(endpoint=project_endpoint, credential=credential) + connections = getattr(client, "connections", None) + list_connections = getattr(connections, "list", None) + if not callable(list_connections): + reason = ( + "AIProjectClient has no connections.list helper " + "(azure-ai-projects too old)." + ) + _store(cache_key, None, reason) + return None, reason + + for connection in list_connections(): + connection_type = str(getattr(connection, "type", "") or "").lower() + target = str(getattr(connection, "target", "") or "").strip() + is_app_insights = ( + "application_insights" in connection_type + or "applicationinsights" in connection_type + ) + is_resource_id = ( + target.lower().startswith("/subscriptions/") + and "/providers/microsoft.insights/components/" in target.lower() + ) + if is_app_insights and is_resource_id: + _store(cache_key, target, None) + return target, None + except Exception as exc: # noqa: BLE001 + reason = _summarize_discovery_exception( + exc, + context="Foundry App Insights connection metadata discovery", + ) + _store(cache_key, None, reason) + return None, reason + + reason = ( + "Foundry returned no Application Insights connection metadata. Wire " + "one in: Project details \u2192 Connected resources \u2192 " + "Add connection \u2192 Application Insights." + ) + _store(cache_key, None, reason) + return None, reason + + def resolve_appinsights_connection_from_env() -> Optional[str]: """Resolve using ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`` if set.""" endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") @@ -300,3 +375,13 @@ def resolve_appinsights_connection_from_env_with_reason() -> Tuple[ if not endpoint: return None, "no AZURE_AI_FOUNDRY_PROJECT_ENDPOINT set" return resolve_appinsights_connection_with_reason(endpoint) + + +def resolve_appinsights_resource_id_from_env_with_reason() -> Tuple[ + Optional[str], Optional[str] +]: + """Resolve App Insights ARM metadata from the configured Foundry project.""" + endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + return None, "no AZURE_AI_FOUNDRY_PROJECT_ENDPOINT set" + return resolve_appinsights_resource_id_with_reason(endpoint) diff --git a/tests/unit/test_agent_identity_service.py b/tests/unit/test_agent_identity_service.py index 147c04d4..c4e4940a 100644 --- a/tests/unit/test_agent_identity_service.py +++ b/tests/unit/test_agent_identity_service.py @@ -249,6 +249,16 @@ def test_resolve_display_name_falls_back_to_existing_record(tmp_path: Path) -> N assert resolve_display_name(tmp_path) == "from-record" +def test_resolve_display_name_from_hosted_agent_url(tmp_path: Path) -> None: + _write_config( + tmp_path, + "agent: https://example.services.ai.azure.com/api/projects/demo/" + "agents/helpdeskbot/versions/13\n", + ) + + assert resolve_display_name(tmp_path) == "helpdeskbot" + + def test_resolve_registration_inputs_returns_name_and_sponsor(tmp_path: Path) -> None: _write_config( tmp_path, diff --git a/tests/unit/test_agent_posture_rules.py b/tests/unit/test_agent_posture_rules.py index fc4698d9..55849e0a 100644 --- a/tests/unit/test_agent_posture_rules.py +++ b/tests/unit/test_agent_posture_rules.py @@ -10,9 +10,6 @@ from agentops.agent.checks.posture_rules.diagnostics import ( evaluate as diagnostics_rule, ) -from agentops.agent.checks.posture_rules.aoai_diagnostic_categories import ( - evaluate as aoai_diag_rule, -) from agentops.agent.checks.posture_rules.local_auth import ( evaluate as local_auth_rule, ) @@ -313,71 +310,4 @@ def test_rule_registry_only_contains_complementary_rules() -> None: "waf.security.local_auth_disabled", "waf.security.managed_identity", "waf.security.diagnostic_settings", - "waf.observability.aoai_diagnostic_categories", } - - -# --------------------------------------------------------------------------- -# aoai_diagnostic_categories (WAF-AI Operational Excellence) -# --------------------------------------------------------------------------- - - -def test_aoai_diag_rule_passes_when_both_categories_enabled() -> None: - payload = _payload( - diagnostic_settings=[ - DiagnosticSettingSnapshot( - name="default", - workspace_id="/subscriptions/s/workspaces/log", - enabled_log_categories=[ - "RequestResponse", - "AzureOpenAIRequestUsage", - ], - ) - ] - ) - assert aoai_diag_rule(payload, "azure_resources") == [] - - -def test_aoai_diag_rule_fires_when_usage_category_missing() -> None: - payload = _payload( - diagnostic_settings=[ - DiagnosticSettingSnapshot( - name="default", - workspace_id="/subscriptions/s/workspaces/log", - enabled_log_categories=["RequestResponse"], - ) - ] - ) - findings = aoai_diag_rule(payload, "azure_resources") - assert len(findings) == 1 - finding = findings[0] - assert finding.id == "waf.observability.aoai_diagnostic_categories" - assert finding.severity is Severity.WARNING - assert finding.category is Category.OPERATIONAL_EXCELLENCE - assert finding.evidence["missing_categories"] == ["AzureOpenAIRequestUsage"] - # The recommendation must carry the exact az fix command. - assert "az monitor diagnostic-settings create" in finding.recommendation - assert "AzureOpenAIRequestUsage" in finding.recommendation - - -def test_aoai_diag_rule_fires_when_no_categories() -> None: - payload = _payload( - diagnostic_settings=[ - DiagnosticSettingSnapshot( - name="default", - workspace_id="/subscriptions/s/workspaces/log", - enabled_log_categories=[], - ) - ] - ) - findings = aoai_diag_rule(payload, "azure_resources") - assert len(findings) == 1 - assert findings[0].evidence["missing_categories"] == [ - "RequestResponse", - "AzureOpenAIRequestUsage", - ] - - -def test_aoai_diag_rule_noop_without_account() -> None: - payload = AzureResourcesPayload(account=None) - assert aoai_diag_rule(payload, "azure_resources") == [] diff --git a/tests/unit/test_agentops_config.py b/tests/unit/test_agentops_config.py index 4e0b30d0..16620b3a 100644 --- a/tests/unit/test_agentops_config.py +++ b/tests/unit/test_agentops_config.py @@ -304,110 +304,6 @@ def test_minimal_config(self, tmp_path) -> None: assert cfg.agent == "my-rag:3" assert cfg.thresholds == {} assert cfg.response_source == "agent" - assert cfg.telemetry_imports == [] - - def test_accepts_telemetry_import_config(self) -> None: - cfg = AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "response_source": "dataset", - "telemetry_imports": [ - { - "name": "prod", - "source": "azure-monitor", - "target": "application-insights", - "resource_id": "$APPINSIGHTS_RESOURCE_ID", - "time_range": {"lookback_days": 14}, - "filters": {"customDimensions.agent": "support"}, - "fields": { - "input": "customDimensions.question", - "response": "customDimensions.answer", - }, - "privacy": {"redact_fields": ["token"], "max_field_length": 500}, - "output": { - "path": ".agentops/data/prod.jsonl", - "label_mode": "pending", - }, - } - ], - } - ) - - item = cfg.telemetry_imports[0] - assert cfg.response_source == "dataset" - assert item.name == "prod" - assert item.source == "azure-monitor" - assert item.target == "application-insights" - assert item.resource_id == "$APPINSIGHTS_RESOURCE_ID" - assert item.time_range.lookback_days == 14 - assert item.output.label_mode == "pending" - - def test_telemetry_import_rejects_unknown_fields(self) -> None: - with pytest.raises(ValidationError): - AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "log-analytics", - "workspace_id": "workspace", - "surprise": True, - } - ], - } - ) - - def test_telemetry_import_time_range_requires_one_mode(self) -> None: - with pytest.raises(ValidationError, match="cannot mix"): - AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "log-analytics", - "workspace_id": "workspace", - "time_range": { - "from": "2026-06-01T00:00:00Z", - "to": "2026-06-02T00:00:00Z", - "lookback_days": 7, - }, - } - ], - } - ) - - def test_telemetry_import_accepts_explicit_time_range(self) -> None: - cfg = AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "log-analytics", - "workspace_id": "workspace", - "time_range": { - "from": "2026-06-01T00:00:00Z", - "to": "2026-06-02T00:00:00Z", - }, - } - ], - } - ) - - time_range = cfg.telemetry_imports[0].time_range - assert time_range.from_ == "2026-06-01T00:00:00Z" - assert time_range.to == "2026-06-02T00:00:00Z" - assert time_range.lookback_days is None def test_resolved_target(self) -> None: cfg = AgentOpsConfig(version=1, agent="my-rag:3", dataset="./qa.jsonl") diff --git a/tests/unit/test_cli_commands.py b/tests/unit/test_cli_commands.py index 3bb6feb7..35b024d0 100644 --- a/tests/unit/test_cli_commands.py +++ b/tests/unit/test_cli_commands.py @@ -1,9 +1,6 @@ from typer.testing import CliRunner from agentops.cli.app import app -from agentops.services.telemetry_import import TelemetryImportPreview - - runner = CliRunner() @@ -49,11 +46,11 @@ def test_eval_help_does_not_expose_compare_subcommand() -> None: assert "compare" not in stripped -def test_planned_command_groups_removed() -> None: - """Stub command groups (monitor/model/dataset/config) are gone in 1.0. +def test_removed_command_groups_are_not_wired() -> None: + """Retired and former stub command groups are absent. `cockpit` is now the real command that opens the local UI.""" - for group in ("monitor", "model", "dataset", "config"): + for group in ("monitor", "model", "dataset", "config", "telemetry"): result = runner.invoke(app, [group, "--help"]) assert result.exit_code != 0, f"unexpected: 'agentops {group}' is still wired" @@ -84,102 +81,3 @@ def test_agent_command_group_wired() -> None: stripped = _strip_ansi(result.stdout) assert "analyze" in stripped assert "serve" in stripped - - -def test_telemetry_validate_uses_named_import(tmp_path, monkeypatch) -> None: - config = tmp_path / "agentops.yaml" - config.write_text( - "\n".join( - [ - "version: 1", - "agent: support-agent:1", - "dataset: .agentops/data/smoke.jsonl", - "telemetry_imports:", - " - name: prod", - " target: log-analytics", - " workspace_id: workspace", - ] - ), - encoding="utf-8", - ) - monkeypatch.setattr( - "agentops.services.telemetry_import.validate_telemetry_import", - lambda _item: [], - ) - - result = runner.invoke(app, ["telemetry", "validate", "prod", "--config", str(config)]) - - assert result.exit_code == 0, result.output - assert "prod" in result.output - assert "valid" in result.output - - -def test_telemetry_preview_prints_service_preview(tmp_path, monkeypatch) -> None: - config = tmp_path / "agentops.yaml" - config.write_text( - "version: 1\n" - "agent: support-agent:1\n" - "dataset: .agentops/data/smoke.jsonl\n" - "telemetry_imports:\n" - " - name: prod\n" - " target: log-analytics\n" - " workspace_id: workspace\n", - encoding="utf-8", - ) - - def fake_preview(item, *, rows=None, apply=False): - return TelemetryImportPreview( - config=item, - output_path=tmp_path / "prod.jsonl", - manifest_path=tmp_path / "prod-manifest.json", - rows=[{"input": "hello", "response": "world"}], - ) - - monkeypatch.setattr( - "agentops.services.telemetry_import.preview_telemetry_import", - fake_preview, - ) - - result = runner.invoke( - app, - ["telemetry", "preview", "prod", "--rows", "1", "--config", str(config)], - ) - - assert result.exit_code == 0, result.output - assert "AgentOps telemetry import" in result.output - assert "hello" in result.output - - -def test_telemetry_import_requires_apply_to_write(tmp_path, monkeypatch) -> None: - config = tmp_path / "agentops.yaml" - config.write_text( - "version: 1\n" - "agent: support-agent:1\n" - "dataset: .agentops/data/smoke.jsonl\n" - "telemetry_imports:\n" - " - name: prod\n" - " target: log-analytics\n" - " workspace_id: workspace\n", - encoding="utf-8", - ) - calls = [] - - def fake_preview(item, *, rows=None, apply=False): - calls.append(apply) - return TelemetryImportPreview( - config=item, - output_path=tmp_path / "prod.jsonl", - manifest_path=tmp_path / "prod-manifest.json", - rows=[], - ) - - monkeypatch.setattr( - "agentops.services.telemetry_import.preview_telemetry_import", - fake_preview, - ) - - result = runner.invoke(app, ["telemetry", "import", "prod", "--config", str(config)]) - - assert result.exit_code == 0, result.output - assert calls == [False] - assert "Dry run only" in result.output diff --git a/tests/unit/test_cli_dashboard.py b/tests/unit/test_cli_dashboard.py deleted file mode 100644 index c064ac23..00000000 --- a/tests/unit/test_cli_dashboard.py +++ /dev/null @@ -1,156 +0,0 @@ -"""CLI tests for the ``agentops telemetry dashboard`` sub-app. - -The Azure logic lives in :mod:`agentops.services.dashboard`; these tests keep -Azure out of the loop by exercising ``deploy --dry-run`` (which never touches -Azure), ``open --print-url`` and ``export`` (both pure), plus the ``--help`` -and ``explain`` surfaces. -""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -from typer.testing import CliRunner - -from agentops.cli.app import app - - -runner = CliRunner() - - -def _strip_ansi(text: str) -> str: - return re.sub(r"\x1b\[[0-9;]*m", "", text) - - -# --------------------------------------------------------------------------- -# help / discoverability -# --------------------------------------------------------------------------- -def test_dashboard_help_lists_commands() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "--help"]) - assert result.exit_code == 0 - stripped = _strip_ansi(result.stdout) - assert "deploy" in stripped - assert "open" in stripped - assert "export" in stripped - assert "agent behavior" in stripped.lower() - - -def test_dashboard_deploy_help() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "deploy", "--help"]) - assert result.exit_code == 0 - stripped = _strip_ansi(result.stdout) - assert "--dry-run" in stripped - assert "--subscription" in stripped - assert "--resource-group" in stripped - assert "--workspace-id" in stripped - assert "--name" in stripped - - -def test_dashboard_open_help() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "open", "--help"]) - assert result.exit_code == 0 - assert "--print-url" in _strip_ansi(result.stdout) - - -def test_dashboard_export_help() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "export", "--help"]) - assert result.exit_code == 0 - assert "--out" in _strip_ansi(result.stdout) - - -# --------------------------------------------------------------------------- -# explain pages -# --------------------------------------------------------------------------- -def test_dashboard_explain_pages_exit_zero() -> None: - for path in ( - ["telemetry", "dashboard", "explain"], - ["telemetry", "dashboard", "deploy", "explain"], - ["telemetry", "dashboard", "open", "explain"], - ["telemetry", "dashboard", "export", "explain"], - ): - result = runner.invoke(app, path) - assert result.exit_code == 0, f"{path} -> {result.stdout}" - - -# --------------------------------------------------------------------------- -# deploy --dry-run (no Azure) -# --------------------------------------------------------------------------- -def test_dashboard_deploy_dry_run_emits_arm_template(tmp_path: Path) -> None: - result = runner.invoke( - app, - [ - "telemetry", - "dashboard", - "deploy", - "--dry-run", - "--subscription", - "sub", - "--resource-group", - "rg", - "--dir", - str(tmp_path), - ], - ) - assert result.exit_code == 0, result.stdout - # The dry-run prints the ARM template to stdout and an advisory note to - # stderr (mixed into stdout by CliRunner); parse just the JSON object. - obj, _end = json.JSONDecoder().raw_decode(result.stdout[result.stdout.index("{") :]) - template = obj - assert template["resources"][0]["type"] == "Microsoft.Insights/workbooks" - assert template["resources"][0]["properties"]["serializedData"] - - -# --------------------------------------------------------------------------- -# open --print-url (no browser, no Azure) -# --------------------------------------------------------------------------- -def test_dashboard_open_print_url(tmp_path: Path) -> None: - result = runner.invoke( - app, - [ - "telemetry", - "dashboard", - "open", - "--print-url", - "--subscription", - "sub", - "--resource-group", - "rg", - "--dir", - str(tmp_path), - ], - ) - assert result.exit_code == 0, result.stdout - assert "https://portal.azure.com/" in result.stdout - - -def test_dashboard_open_falls_back_to_gallery_without_target(tmp_path: Path) -> None: - result = runner.invoke( - app, - ["telemetry", "dashboard", "open", "--print-url", "--dir", str(tmp_path)], - ) - assert result.exit_code == 0, result.stdout - # No subscription/rg discoverable -> gallery deep link. - assert "WorkbookMenuBlade" in result.stdout - - -# --------------------------------------------------------------------------- -# export -# --------------------------------------------------------------------------- -def test_dashboard_export_writes_file(tmp_path: Path) -> None: - out = tmp_path / "foundry-ops.workbook.json" - result = runner.invoke(app, ["telemetry", "dashboard", "export", "--out", str(out)]) - assert result.exit_code == 0, result.stdout - assert out.is_file() - # The exported file is valid JSON identical to the packaged template. - from agentops.services import dashboard as dash - - assert json.loads(out.read_text(encoding="utf-8")) == dash.load_workbook_content() - - -def test_dashboard_export_creates_parent_dirs(tmp_path: Path) -> None: - out = tmp_path / "nested" / "dir" / "wb.json" - result = runner.invoke(app, ["telemetry", "dashboard", "export", "--out", str(out)]) - assert result.exit_code == 0, result.stdout - assert out.is_file() diff --git a/tests/unit/test_cockpit.py b/tests/unit/test_cockpit.py index 5c3a3915..e31ea59a 100644 --- a/tests/unit/test_cockpit.py +++ b/tests/unit/test_cockpit.py @@ -116,88 +116,11 @@ def _write_eval_run( def test_empty_workspace_yields_empty_state(tmp_path: Path): payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - assert payload["eval"]["has_runs"] is False - assert payload["metrics"] == [] assert payload["watchdog"]["has_history"] is False + assert len(payload["readiness"]["checks"]) == 13 html = render_cockpit_html(payload) - assert "No eval runs yet" in html assert "No analysis history yet" in html - assert "agentops eval run" in html - - -def test_cockpit_loads_eval_runs(tmp_path: Path): - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-11T20-00-00Z", - passed=True, - metrics={"coherence": 4.5, "similarity": 4.0, "fluency": 3.7, "f1_score": 0.9}, - ) - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-11T21-00-00Z", - passed=False, - metrics={"coherence": 4.0, "similarity": 3.0, "fluency": 3.0, "f1_score": 0.6}, - target="agent-smoke:3", - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - assert payload["eval"]["has_runs"] is True - eval_keys = {c["key"] for c in payload["eval"]["cards"]} - assert "total_runs" in eval_keys - assert "pass_rate" in eval_keys - assert "latest_run" in eval_keys - # Latest target wins. - latest_card = next(c for c in payload["eval"]["cards"] if c["key"] == "latest_run") - assert latest_card["value"] == "agent-smoke:3" - assert latest_card["badge"]["tone"] == "crit" - - metric_keys = {c["key"] for c in payload["metrics"]} - assert {"coherence", "similarity", "fluency", "f1_score"} <= metric_keys - - -def test_pass_rate_badge_reflects_history(tmp_path: Path): - for i in range(4): - _write_eval_run( - tmp_path, - timestamp_dir=f"2026-05-11T0{i}-00-00Z", - passed=True, - metrics={"coherence": 4.0}, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - pass_card = next(c for c in payload["eval"]["cards"] if c["key"] == "pass_rate") - assert pass_card["value"] == "100%" - assert pass_card["badge"]["tone"] == "ok" - - -def test_metric_trend_badge_detects_regression_for_quality(tmp_path: Path): - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T01-00-00Z", passed=True, - metrics={"coherence": 5.0}, - ) - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T02-00-00Z", passed=True, - metrics={"coherence": 3.0}, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - coh = next(c for c in payload["metrics"] if c["key"] == "coherence") - assert coh["badge"]["label"] == "regressed" - assert coh["badge"]["tone"] == "warn" - - -def test_metric_trend_badge_treats_latency_inversely(tmp_path: Path): - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T01-00-00Z", passed=True, - metrics={"avg_latency_seconds": 5.0}, - ) - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T02-00-00Z", passed=True, - metrics={"avg_latency_seconds": 2.0}, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - lat = next(c for c in payload["metrics"] if c["key"] == "avg_latency_seconds") - # Latency dropping is an improvement, not a regression. - assert lat["badge"]["label"] == "improved" - assert lat["badge"]["tone"] == "ok" + assert "NO-GO" in html def test_telemetry_status_reflects_env(tmp_path: Path, monkeypatch): @@ -216,6 +139,44 @@ def test_telemetry_status_reflects_env(tmp_path: Path, monkeypatch): assert payload["telemetry"]["source"] == "env" +def test_telemetry_status_accepts_foundry_project_managed_identity(monkeypatch): + from agentops.agent.cockpit import _telemetry_status + from agentops.utils import foundry_discovery + + resource_id = ( + "/subscriptions/000/resourceGroups/rg/providers/" + "Microsoft.Insights/components/appi-pmi" + ) + monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) + monkeypatch.delenv("AGENTOPS_APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) + monkeypatch.delenv("AGENTOPS_OTLP_ENDPOINT", raising=False) + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://x.services.ai.azure.com/api/projects/pmi", + ) + monkeypatch.setattr( + foundry_discovery, + "resolve_appinsights_connection_from_env_with_reason", + lambda: ( + None, + "Foundry Application Insights connection uses " + "ProjectManagedIdentity; API Key credentials are not required.", + ), + ) + monkeypatch.setattr( + foundry_discovery, + "resolve_appinsights_resource_id_from_env_with_reason", + lambda: (resource_id, None), + ) + + status = _telemetry_status() + + assert status["enabled"] is True + assert status["source"] == "foundry_project_connection" + assert status["resource_id"] == resource_id + assert status["portal_url"].endswith(f"#resource{resource_id}/overview") + + def test_watchdog_section_surfaces_latest_findings(tmp_path: Path): """The watchdog section exposes the latest run's findings (sorted by severity desc) instead of per-category trend charts.""" @@ -294,7 +255,9 @@ def test_finding_recommendation_renders_safe_markdown(tmp_path: Path): assert "<script>alert(1)</script>" in html -def test_html_includes_all_sections_when_data_present(tmp_path: Path, monkeypatch): +def test_html_contains_exactly_five_sections_in_required_order( + tmp_path: Path, monkeypatch, +): _set_appinsights_env(monkeypatch) _write_eval_run( tmp_path, timestamp_dir="2026-05-11T01-00-00Z", passed=True, @@ -310,122 +273,74 @@ def test_html_includes_all_sections_when_data_present(tmp_path: Path, monkeypatc payload = build_cockpit_payload(tmp_path, time_range=_WIDE) html = render_cockpit_html(payload) - # New strategic sections. - assert "Foundry connection" in html - assert "Foundry launchpad" in html - assert "Azure Monitor" in html - assert "Observability readiness" in html - assert "Next actions" in html - # Consolidated top status cards ("can I ship?"). - assert 'id="section-status-cards"' in html - assert "Can I ship?" in html - # Gate summary sections: eval + quality gate are merged into one - # "Eval gates" section with two subgroups. - assert "Eval gates" in html - assert "Eval gate summary" in html - assert "AgentOps Doctor" in html - assert "CI/CD" in html - assert "Quality gate summary" in html - # Production telemetry is now rendered as a minimal signal into - # Foundry Monitor; the section title is "Production signal". - assert "Production signal" in html - assert "Full view in Foundry Monitor" in html - assert "Fast health snapshot from App Insights" in html - assert "Foundry Monitor for the full production view" in html - assert "Quality gate trends computed from AgentOps result artifacts" in html - assert "AgentOps gate history from local artifacts and CI runs" in html - assert "Foundry connection') status_cards_pos = html.find('id="section-status-cards"') - actions_pos = html.find('Next actions') + connections_pos = html.find('Connections') readiness_pos = html.find('Observability readiness') doctor_pos = html.find('AgentOps Doctor') - eval_gates_pos = html.find('Eval gates') - prod_pos = html.find('Production signal') - deploy_pos = html.find('CI/CD') - launchpad_pos = html.find('Foundry launchpad') - for pos in ( - connection_pos, status_cards_pos, actions_pos, readiness_pos, - doctor_pos, eval_gates_pos, prod_pos, deploy_pos, launchpad_pos, - ): - assert pos != -1, "missing strategic section in cockpit HTML" - assert ( - connection_pos < status_cards_pos < actions_pos < readiness_pos - < doctor_pos < eval_gates_pos < prod_pos < deploy_pos < launchpad_pos + actions_pos = html.find('Next actions') + assert -1 not in ( + status_cards_pos, + connections_pos, + readiness_pos, + doctor_pos, + actions_pos, ) - # Both subgroup labels live inside the merged Eval gates section, eval - # before quality. assert ( - eval_gates_pos - < html.find("Eval gate summary") - < html.find("Quality gate summary") - ) - # Detail sections collapse by default; the top status/next-actions - # sections stay open. At least one collapsed
is present. - assert '
Eval gates', + 'Production signal', + 'CI/CD Pipelines', + 'Foundry launchpad', + "range-pills", + "refreshSelect", + "Auto-refresh", + "window:", + ): + assert removed not in html - The former single-tile "Azure Monitor" group is folded into the - Foundry project subgroup (App Insights + the new Foundry operations - dashboard tile), so there is one place to look instead of a duplicated - one-tile group. - """ - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - open_panel = payload["open_in_foundry"] - groups = open_panel.get("groups") or [] - keys = [g.get("key") for g in groups] - assert keys == ["agent", "project"], ( - "Agent links must precede project links; Azure Monitor is folded " - "into the project group" +def test_connections_only_contains_foundry_and_github(tmp_path: Path, monkeypatch): + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://account.services.ai.azure.com/api/projects/project", ) - - agent_titles = {t["title"] for t in groups[0]["targets"]} - assert {"Agent build", "Monitor", "Traces"}.issubset(agent_titles) - - project_titles = {t["title"] for t in groups[1]["targets"]} - assert { - "Evaluations", - "Datasets", - "Red Teaming", - "Operate overview", - "Foundry operations dashboard", - "App Insights", - }.issubset(project_titles) - - # The new Foundry operations dashboard tile renders right after - # Operate overview, and the two descriptions must not read the same. - project_keys = [t["key"] for t in groups[1]["targets"]] - assert project_keys.index("foundry_ops_dashboard") == ( - project_keys.index("operate") + 1 + monkeypatch.setattr( + "agentops.agent.cockpit._resolve_github_repository", + lambda _workspace: { + "name": "owner/repo", + "url": "https://github.com/owner/repo", + }, ) - descriptions = {t["key"]: t.get("description", "") for t in groups[1]["targets"]} - assert descriptions["operate"] != descriptions["foundry_ops_dashboard"] - + payload = build_cockpit_payload(tmp_path) + items = payload["connections"]["items"] + assert [item["title"] for item in items] == [ + "Foundry project", + "GitHub repository", + ] html = render_cockpit_html(payload) - # Subheaders render (only the two remaining groups). - assert ">Configured agent<" in html - assert ">Foundry project<" in html - # The legacy flat ``targets`` key is kept for backwards-compat and - # combines both groups in display order. - flat_keys = [t["key"] for t in open_panel["targets"]] - assert flat_keys[0] == "agent" - assert flat_keys[-1] == "app_insights" - - -def test_readiness_splits_tracing_and_includes_continuous_eval(tmp_path: Path): - """Readiness now lists separate server-side and client-side tracing - rows plus a dedicated continuous-evaluation row sourced from the - latest Doctor analysis.""" + assert "Open in Foundry" in html + assert "Open in GitHub" in html + assert "Azure tenant" not in html + assert "Application Insights" not in html + + +def test_readiness_splits_connection_and_instrumentation(tmp_path: Path): + """Readiness separates App Insights linkage from agent instrumentation.""" from agentops.agent.cockpit import ( _build_readiness_checklist, _render_readiness_section, @@ -441,8 +356,8 @@ def test_readiness_splits_tracing_and_includes_continuous_eval(tmp_path: Path): tmp_path, telemetry, deployments, watchdog=None, ) titles = [c["title"] for c in readiness["checks"]] - assert any("Server-side tracing" in t for t in titles) - assert any("Client-side tracing" in t for t in titles) + assert "App Insights connection" in titles + assert "Agent tracing instrumentation" in titles cont_row = next( c for c in readiness["checks"] if "Continuous evaluation rules" in c["title"] @@ -452,7 +367,7 @@ def test_readiness_splits_tracing_and_includes_continuous_eval(tmp_path: Path): html = _render_readiness_section(readiness) assert "&rarr;" not in html - assert "Server-side tracing (agent → App Insights)" in html + assert "App Insights connection" in html def test_readiness_detects_multiturn_rubric_sampling_and_replay(tmp_path: Path): @@ -492,6 +407,114 @@ def test_readiness_detects_multiturn_rubric_sampling_and_replay(tmp_path: Path): assert by_title["Trace replay linked to evidence"]["status"] == "ok" +def test_readiness_detects_hosted_otel_eval_rubric_and_unknown_alerts( + tmp_path: Path, +): + from agentops.agent.cockpit import _build_readiness_checklist + + (tmp_path / "azure.yaml").write_text( + "services:\n" + " helpdeskbot:\n" + " host: azure.ai.agent\n" + " kind: hosted\n" + " project: ./src/helpdeskbot\n", + encoding="utf-8", + ) + source = tmp_path / "src" / "helpdeskbot" + source.mkdir(parents=True) + (source / "acs_middleware.py").write_text( + "from opentelemetry import trace\n" + "tracer = trace.get_tracer(__name__)\n" + "with tracer.start_as_current_span('acs'):\n" + " pass\n", + encoding="utf-8", + ) + (source / "eval.yaml").write_text( + "evaluators:\n" + " - name: helpdeskbot-safe-eval\n" + " local_uri: evaluators/helpdeskbot-safe-eval\n", + encoding="utf-8", + ) + + readiness = _build_readiness_checklist( + tmp_path, + { + "enabled": True, + "detail": "Linked through Project Managed Identity.", + "portal_url": "https://portal.azure.com/#resource/appi", + }, + {"has_data": False}, + watchdog=None, + ) + by_title = {check["title"]: check for check in readiness["checks"]} + + assert by_title["App Insights connection"]["status"] == "ok" + tracing = by_title["Agent tracing instrumentation"] + assert tracing["status"] == "ok" + assert "native tracing" in tracing["detail"] + assert "no application-side OpenTelemetry setup is required" in tracing["detail"] + assert "acs_middleware.py" in tracing["detail"] + assert by_title["Optional rubric evaluator gate"]["status"] == "ok" + assert "src/helpdeskbot/eval.yaml" in by_title[ + "Optional rubric evaluator gate" + ]["detail"] + assert by_title["Alerts wired"]["status"] == "info" + assert "Not verified" in by_title["Alerts wired"]["detail"] + assert "does not claim" in by_title["Alerts wired"]["detail"] + + +def test_readiness_recognizes_prompt_agent_native_tracing(tmp_path: Path): + from agentops.agent.cockpit import _build_readiness_checklist + + (tmp_path / "agentops.yaml").write_text( + "version: 1\n" + "agent: support-agent:4\n" + "dataset: .agentops/data/smoke.jsonl\n", + encoding="utf-8", + ) + + readiness = _build_readiness_checklist( + tmp_path, + {"enabled": True, "detail": "Linked", "portal_url": "https://x"}, + {"has_data": False}, + watchdog=None, + ) + tracing = next( + check + for check in readiness["checks"] + if check["title"] == "Agent tracing instrumentation" + ) + + assert tracing["status"] == "ok" + assert "prompt agent runtime" in tracing["detail"] + assert "Custom spans remain optional" in tracing["detail"] + + +def test_readiness_detects_alerts_declared_as_infrastructure(tmp_path: Path): + from agentops.agent.cockpit import _build_readiness_checklist + + infra = tmp_path / "infra" + infra.mkdir() + (infra / "alerts.bicep").write_text( + "resource failedRequests 'Microsoft.Insights/metricAlerts@2018-03-01' = {\n" + " name: 'failed-requests'\n" + "}\n", + encoding="utf-8", + ) + + readiness = _build_readiness_checklist( + tmp_path, + {"enabled": True, "detail": "Linked", "portal_url": "https://x"}, + {"has_data": False}, + watchdog=None, + ) + alerts = next( + check for check in readiness["checks"] if check["title"] == "Alerts wired" + ) + assert alerts["status"] == "ok" + assert "infra/alerts.bicep" in alerts["detail"] + + def test_readiness_non_ready_items_include_remediation(tmp_path: Path, monkeypatch): from agentops.agent.cockpit import _build_readiness_checklist @@ -514,13 +537,16 @@ def test_readiness_non_ready_items_include_remediation(tmp_path: Path, monkeypat assert non_ready for check in non_ready: detail = check["detail"] + if check["title"] == "Alerts wired": + assert "Not verified" in detail + continue assert "How to complete:" in detail assert ("" in detail) or ("Foundry" in detail) by_title = {check["title"]: check["detail"] for check in readiness["checks"]} - assert "OpenTelemetry" in by_title["Client-side tracing (app code instrumented)"] + assert "OpenTelemetry" in by_title["Agent tracing instrumentation"] assert "agentops eval run" in by_title["Scheduled eval (drift watch)"] assert "safe_agent_baseline.yaml" in by_title["Red team scans"] - assert "agentops.eval.*" in by_title["Alerts wired"] + assert "does not claim" in by_title["Alerts wired"] def test_readiness_dots_are_binary_ready_or_not(tmp_path: Path): @@ -580,45 +606,49 @@ def test_readiness_continuous_eval_warns_when_doctor_flags_missing_rules( assert "Foundry monitor docs" in cont_row["detail"] -def test_next_actions_do_not_suggest_workflow_when_ci_gate_exists(tmp_path: Path): - from agentops.agent.cockpit import ( - _build_next_actions, - _build_readiness_checklist, - ) +def test_next_actions_prioritize_doctor_then_incomplete_readiness(): + from agentops.agent.cockpit import _build_next_actions - workflows = tmp_path / ".github" / "workflows" - workflows.mkdir(parents=True) - (workflows / "agentops-pr.yml").write_text( - "name: AgentOps PR\nsteps:\n - run: agentops eval run\n", - encoding="utf-8", - ) - readiness = _build_readiness_checklist( - tmp_path, - {"enabled": True, "detail": "Linked", "portal_url": "https://x"}, - {"has_data": False}, + actions = _build_next_actions( watchdog={ - "has_history": True, "latest_findings": [ { - "id": "safety.config.continuous_eval_missing", + "id": "quality.answer", + "severity": "critical", + "title": "Answer quality is blocked", + "summary": "The response is incomplete.", + "recommendation": "Fix the response policy.", + }, + { + "id": "reliability.trace", "severity": "warning", - } + "title": "Trace coverage is incomplete", + "summary": "A trace is missing.", + "recommendation": "Enable trace capture.", + }, + ], + }, + readiness={ + "checks": [ + { + "title": "Server-side tracing", + "status": "warn", + "detail": "How to complete: enable tracing.", + }, + { + "title": "Alerts wired", + "status": "ok", + "detail": "Ready.", + }, ], }, ) - actions = _build_next_actions( - tmp_path, - {"enabled": True}, - watchdog={"latest_findings": []}, - readiness=readiness, - eval_payload={"runs": [object()]}, - ) - - assert not any( - action["title"] == "Add a CI eval workflow" - for action in actions["actions"] - ) + assert [action["title"] for action in actions["actions"]] == [ + "Fix Doctor: Answer quality is blocked", + "Fix Doctor: Trace coverage is incomplete", + "Complete readiness: Server-side tracing", + ] def test_readiness_detects_official_eval_workflow_and_evidence(tmp_path: Path): @@ -780,43 +810,6 @@ def test_readiness_details_include_azd_eval_and_governance_evidence(tmp_path: Pa assert "Governance evidence: assert: present, acs: present." in detail -def test_cockpit_surfaces_official_eval_artifacts_without_local_runs(tmp_path: Path): - official_dir = tmp_path / ".agentops" / "official-eval" - official_dir.mkdir(parents=True) - (official_dir / "metadata.json").write_text( - json.dumps( - { - "runner": "official-ai-agent-evaluation", - "items_total": 2, - "machine_readable_thresholds": False, - } - ), - encoding="utf-8", - ) - (official_dir / "result.json").write_text( - json.dumps( - { - "runner": "official-ai-agent-evaluation", - "status": "success", - "system": "github-actions", - "machine_readable_thresholds": False, - } - ), - encoding="utf-8", - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert payload["eval"]["has_runs"] is False - assert payload["eval"]["official_eval"]["present"] is True - assert "Official Microsoft Foundry AI Agent Evaluation evidence exists" in html - assert "agentops doctor --evidence-pack" in html - action_titles = [action["title"] for action in payload["next_actions"]["actions"]] - assert "Run your first evaluation" not in action_titles - assert "Generate release evidence" in action_titles - - def test_readiness_detects_prompt_agent_deploy_workflow(tmp_path: Path): from agentops.agent.cockpit import _build_readiness_checklist @@ -866,70 +859,6 @@ def test_readiness_continuous_eval_ok_when_doctor_finds_no_problem( assert cont_row["status"] == "ok" -def test_production_section_is_a_teaser_into_foundry_monitor(tmp_path: Path, monkeypatch): - """Production telemetry now ships as a 2-card teaser (error rate + - P95 latency) with a prominent "Full view in Foundry Monitor" CTA. The - cockpit keeps the quick health snapshot next to the full Foundry - Monitor drilldown.""" - _set_appinsights_env(monkeypatch) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - # Section title reflects the new "signal" framing. - assert "Production signal" in html - assert "Production telemetry" not in html - assert "Fast health snapshot from App Insights" in html - # The skeleton placeholder for the deferred load shows only the - # two surviving teaser cards. - assert "Error rate" in html - assert "P95 latency" in html - assert "Invocations" not in html or "Invocations" in ( - # The token "Invocations" may still appear in unrelated copy - # (e.g. App Insights description in the Foundry connection - # card). Restrict the assertion to the production grid. - "" - ) - - # The production grid renders exactly two skeleton placeholder cards - # (Error rate / P95 latency). Invocations and tokens stay in the full - # Foundry Monitor drilldown. - # The ``skeleton-card`` class is unique to the deferred production - # grid, so counting it across the full HTML is sufficient. - assert html.count("skeleton-card") == 2 - - -def test_production_section_links_to_foundry_monitor_first(tmp_path: Path, monkeypatch): - """The Production signal section must surface the Foundry Monitor - deep-link as a primary CTA so users always know where the full - runtime monitoring surface lives.""" - _set_appinsights_env(monkeypatch) - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-11T01-00-00Z", - passed=True, - metrics={"coherence": 5.0}, - cloud_evaluation={ - "eval_id": "evl_x", - "run_id": "run_x", - "report_url": ( - "https://acct.services.ai.azure.com/api/projects/p/" - "build/evaluations/evl_x" - ), - }, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert "Full view in Foundry Monitor" in html - # The duplicated "Open App Insights KQL" CTA was removed from the - # Production signal section; the App Insights portal URL now lives only - # in the launchpad "App Insights" tile and the Doctor / Eval gate links. - assert "Open App Insights KQL" not in html - # The Foundry Monitor link is styled as the primary section CTA. - assert "section-link-primary" in html - - - def test_deployments_diagnostic_not_a_git_repo(tmp_path: Path): """Empty tempdir → deployments section explains it is not a git repo.""" from agentops.agent.cockpit import ( @@ -998,17 +927,8 @@ def test_create_app_serves_cockpit(tmp_path: Path): assert r.status_code == 200 assert "text/html" in r.headers["content-type"] assert "AgentOps Cockpit" in r.text - # Range bar is present. - assert "range-pills" in r.text - assert 'range=1d' in r.text and 'range=7d' in r.text and 'range=30d' in r.text - - # Custom range round-trip (also returns shell unless _partial is set). - r = client.get("/?range=custom&from=2020-01-01&to=2030-01-01&_partial=1") - assert r.status_code == 200 - - # Unknown range falls back to 7d default. - r = client.get("/?range=eternity&_partial=1") - assert r.status_code == 200 + assert "range-pills" not in r.text + assert "refreshSelect" not in r.text r = client.get("/healthz") assert r.status_code == 200 @@ -1218,68 +1138,6 @@ def test_foundry_deeplinks_use_only_build_routes(tmp_path): assert links["operate"].split("?")[0].endswith("/operate/overview") -def test_foundry_dataset_card_explains_inline_eval_data(tmp_path: Path): - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-12T22-30-00Z", - passed=True, - metrics={"similarity": 0.9}, - cloud_evaluation={ - "report_url": ( - "https://ai.azure.com/nextgen/r/" - "abc123,rg-x,,acct-y,proj-z/build/evaluations/" - "eval_001/run/run_001" - ), - "dataset": { - "mode": "inline", - "requested_mode": "auto", - "source_type": "file_content", - "local_path": ".agentops/data/smoke.jsonl", - "foundry_behavior": ( - "Foundry may materialize inline rows as eval-data-* " - "backing dataset assets." - ), - }, - }, - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert "Latest cloud run used local JSONL inline" in html - assert "eval-data-*" in html - - -def test_foundry_dataset_card_shows_synced_dataset(tmp_path: Path): - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-12T22-31-00Z", - passed=True, - metrics={"similarity": 0.9}, - cloud_evaluation={ - "report_url": ( - "https://ai.azure.com/nextgen/r/" - "abc123,rg-x,,acct-y,proj-z/build/evaluations/" - "eval_001/run/run_001" - ), - "dataset": { - "mode": "foundry", - "requested_mode": "auto", - "source_type": "file_id", - "local_path": ".agentops/data/smoke.jsonl", - "foundry_name": "agentops-smoke", - "foundry_version": "sha256-abc123", - }, - }, - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert "Latest cloud run used Foundry dataset agentops-smoke@sha256-abc123" in html - assert "eval-data-*" not in html - - def test_readiness_detail_links_use_info_color(tmp_path: Path, monkeypatch): monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) monkeypatch.delenv("AGENTOPS_APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) @@ -1303,23 +1161,6 @@ def test_doctor_section_has_no_foundry_control_plane_link(tmp_path): assert "Open Foundry control plane" not in html -def test_tenant_card_source_moves_to_tooltip(tmp_path, monkeypatch): - """The "(from az account show)" suffix used to live inline in the - tenant card detail. It is now an ``(i)`` hover tooltip so the card - body stays compact.""" - # Force the tenant detection to a known value without invoking az. - monkeypatch.setattr( - "agentops.agent.cockpit._az_tenant_id", - lambda: "16b3c013-d300-468d-ac64-7eda0820b6d3", - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - # Inline source text is gone. - assert "(from az account show)" not in html - # Tooltip surfaces the same source via title= attribute. - assert "Resolved from `az account show`." in html - - def test_tenant_lookup_allows_slow_az_cmd_cold_start(monkeypatch): """Windows az.cmd can take several seconds on the first call. The Cockpit should wait long enough to resolve the tenant instead of @@ -1347,26 +1188,6 @@ def fake_run(*args, **kwargs): cockpit._TENANT_CACHE.clear() -def test_app_insights_card_source_moves_to_tooltip( - tmp_path, monkeypatch -): - """The "Connected via APPLICATIONINSIGHTS_CONNECTION_STRING" detail - used to render inline. It is now an ``(i)`` hover tooltip on the - App Insights card.""" - monkeypatch.setenv( - "APPLICATIONINSIGHTS_CONNECTION_STRING", - "InstrumentationKey=00000000-0000-0000-0000-000000000000;" - "IngestionEndpoint=https://example.in.applicationinsights.azure.com/", - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - # The verbose inline message is gone. - assert "Connected via APPLICATIONINSIGHTS_CONNECTION_STRING" not in html - # The tooltip carries the env-var reference. - assert "APPLICATIONINSIGHTS_CONNECTION_STRING" in html - assert "info-i" in html - - def test_app_insights_logs_query_is_bounded(monkeypatch): from agentops.agent.cockpit import _appinsights_portal_url @@ -1439,7 +1260,7 @@ def test_app_insights_eval_runs_query_and_link(monkeypatch, tmp_path): payload = build_cockpit_payload(tmp_path, time_range=_WIDE) html = render_cockpit_html(payload) - assert "View CI evals in App Insights" in html + assert "View CI evals in App Insights" not in html def test_foundry_project_card_compacts_endpoint_and_exposes_copy(tmp_path, monkeypatch): diff --git a/tests/unit/test_dashboard.py b/tests/unit/test_dashboard.py deleted file mode 100644 index 8bca2861..00000000 --- a/tests/unit/test_dashboard.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Unit tests for the Foundry operations dashboard service. - -These cover the pure, Azure-free surface of -:mod:`agentops.services.dashboard`: template loading, the portal URL builder -(deployed vs. gallery fallback), the diagnostic-settings helpers, the ARM -template shape, and the RBAC preflight messaging with the authorization -helpers mocked. -""" - -from __future__ import annotations - -import json -from importlib.resources import files as package_files -from pathlib import Path - -import pytest - -from agentops.services import dashboard as dash - - -_FIXTURES = Path(__file__).parents[1] / "fixtures" -_EVALUATOR_KEYS = ("gen_ai.evaluation.name", "evaluator") -_SCORE_KEYS = ( - "gen_ai.evaluation.score.value", - "gen_ai.evaluation.score", - "score", -) -_LABEL_KEYS = ( - "gen_ai.evaluation.score.label", - "gen_ai.evaluation.result", - "label", -) - - -def _first_property(properties: dict[str, object], keys: tuple[str, ...]) -> str: - for key in keys: - value = properties.get(key) - if value not in (None, ""): - return str(value) - return "" - - -def _fixture_projection(event: dict[str, object]) -> dict[str, object]: - properties = event["properties"] - assert isinstance(properties, dict) - evaluator = _first_property(properties, _EVALUATOR_KEYS) - score_text = _first_property(properties, _SCORE_KEYS) - label = _first_property(properties, _LABEL_KEYS) - try: - numeric_score = float(score_text) if score_text else None - except ValueError: - numeric_score = None - agent_id = str(properties.get("gen_ai.agent.id", "")) - version = str(properties.get("gen_ai.agent.version", "")) - if not version and ":" in agent_id: - version = agent_id.split(":", 1)[1] - return { - "recognized": bool(evaluator or score_text or label), - "numeric_score": numeric_score, - "version": version or "Version not reported", - "raw_properties": properties, - } - - -# --------------------------------------------------------------------------- -# Template loading -# --------------------------------------------------------------------------- -def test_load_workbook_template_returns_valid_json() -> None: - raw = dash.load_workbook_template() - assert isinstance(raw, str) and raw.strip() - parsed = json.loads(raw) - # The packaged asset is a gallery-template workbook. - assert isinstance(parsed, dict) - - -def test_load_workbook_content_matches_template() -> None: - assert dash.load_workbook_content() == json.loads(dash.load_workbook_template()) - - -def test_agent_behavior_tab_is_additive_and_preserves_existing_navigation() -> None: - content = dash.load_workbook_content() - tabs = next(item for item in content["items"] if item["name"] == "tabs") - labels = [link["linkLabel"] for link in tabs["content"]["links"]] - assert labels == [ - "Capacity", - "Traffic and tokens", - "Latency", - "Errors and throttling", - "Agent behavior", - ] - groups = {item["name"]: item for item in content["items"] if item["type"] == 12} - assert { - "group-capacity", - "group-traffic", - "group-latency", - "group-errors", - }.issubset(groups) - behavior = groups["group-agent-behavior"] - assert behavior["conditionalVisibility"]["value"] == "agent-behavior" - - -def test_agent_behavior_tab_surfaces_states_filters_and_preview_boundary() -> None: - content = dash.load_workbook_content() - behavior = next( - item for item in content["items"] if item["name"] == "group-agent-behavior" - ) - items = behavior["content"]["items"] - note = next(item for item in items if item["name"] == "agent-behavior-note") - note_text = note["content"]["json"] - for state in ( - "Schema unavailable", - "No access", - "No data", - "Filter empty", - "Possible ingestion delay", - ): - assert state in note_text - assert "Preview" in note_text - assert "Foundry" in note_text - assert "does not create, schedule, gate, or edit evaluations" in note_text - assert "human trace annotations" in note_text - assert "validation-dependent" in note_text - assert "does not require `gen_ai.agent.id`" in note_text - - filters = next(item for item in items if item["name"] == "agent-behavior-filters") - assert [parameter["name"] for parameter in filters["content"]["parameters"]] == [ - "AgentEnvironment", - "AgentName", - "AgentVersion", - "Evaluator", - ] - - -def test_agent_behavior_queries_use_bounded_versioned_normalization() -> None: - content = dash.load_workbook_content() - behavior = next( - item for item in content["items"] if item["name"] == "group-agent-behavior" - ) - query_items = [ - item - for item in behavior["content"]["items"] - if item["type"] == 3 and "query" in item["content"] - ] - assert len(query_items) == 7 - assert all( - item["content"]["timeContextFromParameter"] == "TimeRange" - for item in query_items - ) - combined = "\n".join(item["content"]["query"] for item in query_items) - for fragment in ( - "set best_effort=true", - "union isfuzzy=true", - "AppEvents", - "customEvents", - "Name == 'gen_ai.evaluation.result'", - "name == 'gen_ai.evaluation.result'", - "Properties", - "customDimensions", - "gen_ai.evaluation.score.value", - "gen_ai.evaluation.score.label", - "Version not reported", - "RawProperties", - ): - assert fragment in combined - status = next( - item for item in query_items if item["name"] == "agent-behavior-status" - ) - status_query = status["content"]["query"] - assert "agent_behavior/v1" in status_query - assert "ObservedInvokeAgentInvocations" in status_query - assert "EvaluatedTraces" in status_query - assert "EvaluationEvents" in status_query - assert "Coverage" not in status_query - assert "automated trace-evaluation export validation-dependent" in status_query - schema_diagnostics = next( - item - for item in query_items - if item["name"] == "agent-behavior-schema-diagnostics" - ) - assert "RawProperties" in schema_diagnostics["content"]["query"] - assert "| take 100" in schema_diagnostics["content"]["query"] - - -def test_agent_behavior_does_not_combine_unlike_raw_score_scales() -> None: - content = dash.load_workbook_content() - behavior = next( - item for item in content["items"] if item["name"] == "group-agent-behavior" - ) - items = {item["name"]: item for item in behavior["content"]["items"]} - assert items["agent-behavior-score-trend"]["content"]["visualization"] == "table" - assert items["agent-behavior-pass-trend"]["content"]["visualization"] == "timechart" - assert ( - items["agent-behavior-volume-trend"]["content"]["visualization"] == "timechart" - ) - assert ( - "do not compare evaluators" - in items["agent-behavior-score-trend"]["content"]["title"] - ) - - -def test_agent_behavior_authoring_query_is_packaged_and_bounded() -> None: - resource = ( - package_files("agentops.templates") - .joinpath("workbooks/queries/agent_behavior.kql") - .read_text(encoding="utf-8") - ) - assert "agent_behavior/v1" in resource - assert "between (_startTime .. _endTime)" in resource - assert "AppEvents" in resource and "customEvents" in resource - assert "AppDependencies" in resource and "dependencies" in resource - assert "RawProperties" in resource - assert "human trace" in resource and "annotations" in resource - assert "Automated trace-evaluation export" in resource - assert "validation-dependent" in resource - - -def test_agent_behavior_schema_fixtures_cover_supported_shapes_and_edges() -> None: - events = json.loads( - (_FIXTURES / "workbook_agent_behavior_events.json").read_text(encoding="utf-8") - ) - assert {event["source_table"] for event in events} == { - "AppEvents", - "customEvents", - } - expected_columns = { - "AppEvents": ("Name", "Properties"), - "customEvents": ("name", "customDimensions"), - } - projections = {} - for event in events: - name_column, properties_column = expected_columns[event["source_table"]] - assert event["name_column"] == name_column - assert event["properties_column"] == properties_column - assert event["event_name"] == "gen_ai.evaluation.result" - projection = _fixture_projection(event) - projections[event["case"]] = projection - assert projection["recognized"] is event["expected"]["recognized"] - assert projection["numeric_score"] == event["expected"]["numeric_score"] - assert projection["version"] == event["expected"]["version"] - assert projection["raw_properties"] == event["properties"] - - assert projections["missing_optional_fields"]["version"] == "Version not reported" - assert projections["nonnumeric_score"]["numeric_score"] is None - assert projections["unrecognized_schema"]["recognized"] is False - evaluators = { - _first_property(event["properties"], _EVALUATOR_KEYS) - for event in events - if event["expected"]["recognized"] - } - assert {"Relevance", "Groundedness", "IntentResolution", "Fluency"} <= evaluators - same_trace = [ - event for event in events if event["properties"].get("trace_id") == "trace-2" - ] - assert { - _first_property(event["properties"], _EVALUATOR_KEYS) for event in same_trace - } == {"Groundedness", "Fluency"} - - -# --------------------------------------------------------------------------- -# Portal URL -# --------------------------------------------------------------------------- -def test_make_workbook_resource_id_is_deterministic() -> None: - a = dash.make_workbook_resource_id("sub", "rg", "AgentOps Foundry operations") - b = dash.make_workbook_resource_id("sub", "rg", "AgentOps Foundry operations") - assert a == b - assert a.startswith("/subscriptions/sub/resourceGroups/rg/providers/") - assert dash.WORKBOOK_RESOURCE_TYPE in a - - -def test_portal_url_deployed_when_target_known() -> None: - url = dash.build_workbook_portal_url( - subscription_id="sub", - resource_group="rg", - name="AgentOps Foundry operations", - tenant_id="tenant-123", - ) - assert url.startswith("https://portal.azure.com/#@tenant-123/resource") - assert url.endswith("/workbook") - - -def test_portal_url_deployed_without_tenant() -> None: - url = dash.build_workbook_portal_url( - subscription_id="sub", resource_group="rg", name="wb" - ) - assert url.startswith("https://portal.azure.com/#/resource") - assert "@" not in url.split("/resource", 1)[0] - - -def test_portal_url_falls_back_to_gallery_when_target_unknown() -> None: - url = dash.build_workbook_portal_url() - assert "WorkbookMenuBlade" in url - assert url.endswith("/gallery") - - -def test_portal_url_prefers_explicit_resource_id() -> None: - rid = "/subscriptions/s/resourceGroups/g/providers/x/y/z" - url = dash.build_workbook_portal_url(workbook_resource_id=rid, tenant_id="t") - assert url == f"https://portal.azure.com/#@t/resource{rid}/workbook" - - -# --------------------------------------------------------------------------- -# Diagnostic settings -# --------------------------------------------------------------------------- -def test_missing_diagnostic_categories_none_when_all_present() -> None: - assert ( - dash.missing_diagnostic_categories( - ["RequestResponse", "AzureOpenAIRequestUsage", "Audit"] - ) - == [] - ) - - -def test_missing_diagnostic_categories_reports_absent() -> None: - assert dash.missing_diagnostic_categories(["RequestResponse"]) == [ - "AzureOpenAIRequestUsage" - ] - assert dash.missing_diagnostic_categories([]) == list( - dash.REQUIRED_DIAGNOSTIC_CATEGORIES - ) - - -def test_build_diagnostic_settings_command_shape() -> None: - cmd = dash.build_diagnostic_settings_command( - aoai_resource_id="/subscriptions/s/aoai", - workspace_id="/subscriptions/s/ws", - ) - assert cmd.startswith("az monitor diagnostic-settings create ") - assert "--resource /subscriptions/s/aoai" in cmd - assert "--workspace /subscriptions/s/ws" in cmd - assert "RequestResponse" in cmd and "AzureOpenAIRequestUsage" in cmd - - -def test_build_diagnostic_settings_command_uses_placeholders() -> None: - cmd = dash.build_diagnostic_settings_command( - aoai_resource_id=None, workspace_id=None - ) - assert "" in cmd - assert "" in cmd - - -# --------------------------------------------------------------------------- -# ARM template -# --------------------------------------------------------------------------- -def test_build_arm_template_shape() -> None: - target = dash.DashboardTarget( - name="AgentOps Foundry operations", - subscription_id="sub", - resource_group="rg", - workspace_id="/subscriptions/sub/ws", - ) - template = dash.build_arm_template(target=target) - assert template["$schema"].startswith("https://schema.management.azure.com") - assert len(template["resources"]) == 1 - resource = template["resources"][0] - assert resource["type"] == dash.WORKBOOK_RESOURCE_TYPE - assert resource["properties"]["displayName"] == "AgentOps Foundry operations" - # serializedData is the workbook content re-serialized as a string. - assert json.loads(resource["properties"]["serializedData"]) == ( - dash.load_workbook_content() - ) - assert resource["properties"]["sourceId"] == "/subscriptions/sub/ws" - - -def test_build_arm_template_location_defaults_to_resource_group_expression() -> None: - # Workbooks reject "global"; the template must resolve to the RG region. - target = dash.DashboardTarget(name="wb", subscription_id="sub", resource_group="rg") - template = dash.build_arm_template(target=target) - assert template["resources"][0]["location"] == "[resourceGroup().location]" - - -def test_build_arm_template_location_honors_explicit_region() -> None: - target = dash.DashboardTarget( - name="wb", subscription_id="sub", resource_group="rg", location="eastus2" - ) - template = dash.build_arm_template(target=target) - assert template["resources"][0]["location"] == "eastus2" - - -def test_build_arm_template_name_matches_resource_id() -> None: - target = dash.DashboardTarget(name="wb", subscription_id="sub", resource_group="rg") - template = dash.build_arm_template(target=target) - rid = dash.make_workbook_resource_id("sub", "rg", "wb") - assert template["resources"][0]["name"] == rid.rsplit("/", 1)[-1] - - -# --------------------------------------------------------------------------- -# RBAC preflight -# --------------------------------------------------------------------------- -def test_check_rbac_errors_without_subscription() -> None: - result = dash.check_rbac( - subscription_id=None, resource_group="rg", workspace_id=None - ) - assert result.ok is False - assert result.level == "error" - assert any("subscription" in m.lower() for m in result.messages) - - -def _patch_rbac(monkeypatch, *, rg_roles, ws_roles=None, raise_error=None): - """Install fakes for the lazy-imported authorization helpers.""" - import agentops.agent.checks._rbac_authorization as rbac_mod - - class _AuthErr(Exception): - pass - - monkeypatch.setattr(rbac_mod, "AuthorizationCheckError", _AuthErr) - monkeypatch.setattr( - rbac_mod, "resolve_signed_in_principal_object_id", lambda: "principal-oid" - ) - - def _list(*, subscription_id, scope, principal_object_id): - if raise_error is not None: - raise _AuthErr(raise_error) - if scope.endswith("/resourceGroups/rg"): - return list(rg_roles) - return list(ws_roles or []) - - monkeypatch.setattr(rbac_mod, "list_principal_role_definition_ids", _list) - return _AuthErr - - -def test_check_rbac_passes_when_roles_present(monkeypatch) -> None: - _patch_rbac( - monkeypatch, - rg_roles=[dash._ROLE_WORKBOOK_CONTRIBUTOR], - ws_roles=[dash._ROLE_LOG_ANALYTICS_READER], - ) - result = dash.check_rbac( - subscription_id="sub", - resource_group="rg", - workspace_id="/subscriptions/sub/ws", - ) - assert result.ok is True - assert result.level == "ok" - assert any("passed" in m.lower() for m in result.messages) - - -def test_check_rbac_fails_when_workbook_role_missing(monkeypatch) -> None: - _patch_rbac(monkeypatch, rg_roles=[dash._ROLE_READER], ws_roles=[]) - result = dash.check_rbac( - subscription_id="sub", resource_group="rg", workspace_id=None - ) - assert result.ok is False - assert result.level == "error" - joined = " ".join(result.messages) - assert "Workbook Contributor" in joined - assert "'rg'" in joined - assert "--dry-run" in joined - - -def test_check_rbac_fails_when_workspace_role_missing(monkeypatch) -> None: - _patch_rbac( - monkeypatch, - rg_roles=[dash._ROLE_WORKBOOK_CONTRIBUTOR], - ws_roles=[dash._ROLE_READER.replace("a", "z")], # unrelated role guid - ) - result = dash.check_rbac( - subscription_id="sub", - resource_group="rg", - workspace_id="/subscriptions/sub/ws", - ) - assert result.ok is False - assert any("Log Analytics Reader" in m for m in result.messages) - - -def test_check_rbac_fails_open_on_authorization_error(monkeypatch) -> None: - _patch_rbac(monkeypatch, rg_roles=[], raise_error="listing denied") - result = dash.check_rbac( - subscription_id="sub", resource_group="rg", workspace_id=None - ) - # Fails OPEN: warn but allow deploy to proceed. - assert result.ok is True - assert result.level == "warn" - assert any("listing denied" in m for m in result.messages) - - -def test_check_rbac_fails_open_when_helpers_unavailable(monkeypatch) -> None: - import builtins - - real_import = builtins.__import__ - - def _blocked(name, *args, **kwargs): - if name == "agentops.agent.checks._rbac_authorization": - raise ImportError("no authorization helpers") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", _blocked) - result = dash.check_rbac( - subscription_id="sub", resource_group="rg", workspace_id=None - ) - assert result.ok is True - assert result.level == "warn" - - -# --------------------------------------------------------------------------- -# deploy_workbook guards (no live Azure) -# --------------------------------------------------------------------------- -def test_deploy_workbook_requires_subscription() -> None: - target = dash.DashboardTarget(name="wb") - with pytest.raises(dash.DashboardError) as exc: - dash.deploy_workbook(target=target) - assert "subscription" in str(exc.value).lower() - - -def test_deploy_workbook_errors_when_az_missing(monkeypatch) -> None: - monkeypatch.setattr(dash, "_az_executable", lambda: None) - target = dash.DashboardTarget(name="wb", subscription_id="sub", resource_group="rg") - with pytest.raises(dash.DashboardError) as exc: - dash.deploy_workbook(target=target) - assert "az" in str(exc.value).lower() diff --git a/tests/unit/test_foundry_discovery.py b/tests/unit/test_foundry_discovery.py index 235ba1a5..c5896abb 100644 --- a/tests/unit/test_foundry_discovery.py +++ b/tests/unit/test_foundry_discovery.py @@ -272,6 +272,39 @@ def test_with_reason_accepts_project_managed_identity_connection(): assert reason == PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON +def test_resource_id_discovery_accepts_project_managed_identity_metadata(): + resource_id = ( + "/subscriptions/000/resourceGroups/rg/providers/" + "Microsoft.Insights/components/appi" + ) + connection = mock.Mock() + connection.type = "ConnectionType.APPLICATION_INSIGHTS" + connection.target = resource_id + fake_connections = mock.MagicMock() + fake_connections.list.return_value = iter([connection]) + fake_client = mock.MagicMock() + fake_client.connections = fake_connections + fake_projects_mod = mock.MagicMock() + fake_projects_mod.AIProjectClient.return_value = fake_client + fake_identity_mod = mock.MagicMock() + + with mock.patch.dict( + "sys.modules", + {"azure.ai.projects": fake_projects_mod, "azure.identity": fake_identity_mod}, + ): + from agentops.utils.foundry_discovery import ( + resolve_appinsights_resource_id_with_reason, + ) + + result, reason = resolve_appinsights_resource_id_with_reason( + "https://x.services.ai.azure.com/api/projects/pmi" + ) + + assert result == resource_id + assert reason is None + fake_connections.list.assert_called_once_with() + + def test_with_reason_reports_missing_app_insights_connection(): class ResourceNotFoundError(Exception): pass diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index 926717e5..fcb1c9cb 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -581,6 +581,62 @@ def query_resource( assert payload.p95_duration_seconds == 2.5 +def test_azure_monitor_uses_foundry_app_insights_resource_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from agentops.utils import foundry_discovery + + resource_id = ( + "/subscriptions/000/resourceGroups/rg/providers/" + "Microsoft.Insights/components/appi-pmi" + ) + captured: dict[str, object] = {} + + class LogsQueryStatus: + FAILURE = "Failure" + + class Response: + status = "Success" + tables: list[object] = [] + + class LogsQueryClient: + def __init__(self, _credential: object) -> None: + pass + + def query_resource( + self, + *, + resource_id: str, + query: str, + timespan: object, + ) -> Response: + captured["resource_id"] = resource_id + return Response() + + identity_module = types.ModuleType("azure.identity") + identity_module.DefaultAzureCredential = object # type: ignore[attr-defined] + query_module = types.ModuleType("azure.monitor.query") + query_module.LogsQueryClient = LogsQueryClient # type: ignore[attr-defined] + query_module.LogsQueryStatus = LogsQueryStatus # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "azure.identity", identity_module) + monkeypatch.setitem(sys.modules, "azure.monitor.query", query_module) + monkeypatch.setattr( + foundry_discovery, + "resolve_appinsights_resource_id_from_env_with_reason", + lambda: (resource_id, None), + ) + + payload = azure_monitor.collect_azure_monitor( + AzureMonitorSourceConfig(enabled=True), + lookback_days=7, + ) + + assert captured["resource_id"] == resource_id + assert payload.diagnostics["target"] == resource_id + assert payload.diagnostics["target_source"] == "foundry_project_connection" + assert payload.diagnostics["status"] == "ok" + + def test_azure_monitor_uses_connection_string_application_id( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_telemetry_import.py b/tests/unit/test_telemetry_import.py deleted file mode 100644 index aa07010f..00000000 --- a/tests/unit/test_telemetry_import.py +++ /dev/null @@ -1,153 +0,0 @@ -from __future__ import annotations - -import builtins -import json - -import pytest - -from agentops.core.agentops_config import AgentOpsConfig -from agentops.services.telemetry_import import ( - TelemetryImportError, - build_telemetry_kql, - find_telemetry_import, - query_azure_monitor, - transform_telemetry_rows, - write_telemetry_import, -) - - -def _config(**overrides): - data = { - "version": 1, - "agent": "support-agent:1", - "dataset": ".agentops/data/smoke.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "application-insights", - "resource_id": "$APPINSIGHTS_RESOURCE_ID", - "fields": { - "input": "customDimensions.question", - "response": "customDimensions.answer", - "context": "customDimensions.context", - }, - "output": {"path": ".agentops/data/prod.jsonl"}, - **overrides, - } - ], - } - return AgentOpsConfig.model_validate(data).telemetry_imports[0] - - -def test_transform_rows_dedupes_redacts_and_writes_manifest(tmp_path) -> None: - cfg = _config( - output={"path": str(tmp_path / "prod.jsonl")}, - privacy={"redact_fields": ["token"], "max_field_length": 100, "include_raw": True}, - ) - raw = [ - { - "operation_Id": "trace-1", - "id": "turn-1", - "customDimensions": { - "question": "How do I reset my password?", - "answer": "Open account settings.", - "context": "Reset article", - "token": "secret-token", - }, - }, - { - "operation_Id": "trace-1", - "id": "turn-1", - "customDimensions": { - "question": "How do I reset my password?", - "answer": "Open account settings.", - }, - }, - {"customDimensions": {"question": "missing response"}}, - ] - - preview = transform_telemetry_rows(cfg, raw) - write_telemetry_import(preview) - - assert len(preview.rows) == 1 - assert preview.deduped == 1 - assert preview.skipped == 1 - row = preview.rows[0] - assert row["input"] == "How do I reset my password?" - assert row["response"] == "Open account settings." - assert row["expected"] == "Open account settings." - assert row["context"] == "Reset article" - assert row["telemetry"]["trace_id"] == "trace-1" - assert row["raw"]["customDimensions"]["token"] == "[redacted]" - assert (tmp_path / "prod.jsonl").exists() - manifest = json.loads((tmp_path / "prod-manifest.json").read_text(encoding="utf-8")) - assert manifest["rows"] == 1 - assert manifest["deduped"] == 1 - - -def test_build_kql_uses_safe_generated_filters() -> None: - cfg = _config(filters={"customDimensions.agent": ["support", "sales"]}, max_rows=1000) - - kql = build_telemetry_kql(cfg, rows=5) - - assert "union isfuzzy=true requests, dependencies, traces" in kql - assert "| extend timestamp = coalesce(" in kql - assert "column_ifexists('timestamp', datetime(null))" in kql - assert "column_ifexists('TimeGenerated', datetime(null))" in kql - assert "coalesce(timestamp, TimeGenerated)" not in kql - assert "ago(7d)" in kql - assert ( - "tostring(column_ifexists('customDimensions', dynamic({}))['agent']) " - "in ('support', 'sales')" - ) in kql - assert "operation_Id = column_ifexists('operation_Id', '')" in kql - assert "TimeGenerated =" not in kql - assert "| order by timestamp desc" in kql - assert "take 5" in kql - - -def test_build_kql_guards_plain_filter_columns() -> None: - cfg = _config(filters={"name": "agent.response"}) - - kql = build_telemetry_kql(cfg, rows=10) - - assert "tostring(column_ifexists('name', '')) == 'agent.response'" in kql - assert "tostring(name)" not in kql - - -def test_build_kql_rejects_unsafe_filter_field() -> None: - cfg = _config(filters={"name); drop table traces; //": "x"}) - - with pytest.raises(TelemetryImportError, match="unsafe"): - build_telemetry_kql(cfg) - - -def test_find_telemetry_import_reports_available_names() -> None: - cfg = AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "support-agent:1", - "dataset": ".agentops/data/smoke.jsonl", - "telemetry_imports": [ - {"name": "prod", "target": "log-analytics", "workspace_id": "workspace"} - ], - } - ) - - with pytest.raises(TelemetryImportError, match="prod"): - find_telemetry_import(cfg, "missing") - - -def test_query_azure_monitor_reports_missing_sdk(monkeypatch) -> None: - cfg = _config() - original_import = builtins.__import__ - - def fake_import(name, *args, **kwargs): - if name == "azure.identity": - raise ImportError("no azure") - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", fake_import) - - with pytest.raises(TelemetryImportError, match="azure-identity"): - query_azure_monitor(cfg, rows=1)