From 3548a0d13b453d529b740cc2642b9f9a2ec4bf44 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Thu, 13 Aug 2026 22:07:48 -0300 Subject: [PATCH 1/8] fix: pass built version to dev verification Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 70 +++++++++++++++++++++++++--------- tests/unit/test_ci_workflow.py | 43 +++++++++++++++++++++ 2 files changed, 94 insertions(+), 19 deletions(-) create mode 100644 tests/unit/test_ci_workflow.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ea0aff..7bd2965 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,6 +149,8 @@ jobs: needs: [lint, test] runs-on: ubuntu-latest environment: staging + outputs: + version: ${{ steps.package-version.outputs.version }} permissions: id-token: write # Required for PyPI Trusted Publishing (OIDC) steps: @@ -171,10 +173,44 @@ jobs: - name: Build package run: uv build + - name: Determine built version + id: package-version + run: | + VERSION=$(uv run python - <<'PY' + from email.parser import BytesParser + from pathlib import Path + from zipfile import ZipFile + + wheels = sorted(Path("dist").glob("*.whl")) + if len(wheels) != 1: + raise SystemExit( + f"Expected exactly one wheel in dist/, found {len(wheels)}." + ) + with ZipFile(wheels[0]) as archive: + metadata_files = [ + name + for name in archive.namelist() + if name.endswith(".dist-info/METADATA") + ] + if len(metadata_files) != 1: + raise SystemExit( + "Expected exactly one .dist-info/METADATA file in " + f"{wheels[0].name}, found {len(metadata_files)}." + ) + metadata = BytesParser().parsebytes(archive.read(metadata_files[0])) + version = metadata.get("Version") + if not version: + raise SystemExit(f"No Version field found in {wheels[0].name}.") + print(version) + PY + ) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Built package version: $VERSION" + - name: Show version run: | ls -la dist/ - uv run python -c "from importlib.metadata import version; print(f'Dev version: {version(\"agentops-accelerator\")}')" + echo "Dev version: ${{ steps.package-version.outputs.version }}" - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 @@ -189,33 +225,27 @@ jobs: needs: publish-dev runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Set up Python uses: actions/setup-python@v7 with: python-version: "3.12" - - name: Determine expected version - id: version - run: | - pip install setuptools-scm - VERSION=$(python -m setuptools_scm) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Expected dev version: $VERSION" - - name: Install from TestPyPI + env: + PACKAGE_VERSION: ${{ needs.publish-dev.outputs.version }} run: | + if [ -z "$PACKAGE_VERSION" ]; then + echo "::error::publish-dev did not expose the built package version." + exit 1 + fi # TestPyPI serves its simple index through a CDN, so a freshly # uploaded release is not immediately resolvable even after the # upload returns 200 OK. Allow ~6 minutes for it to propagate. ATTEMPTS=12 for i in $(seq 1 "$ATTEMPTS"); do - echo "Attempt $i/$ATTEMPTS: installing agentops-accelerator==${{ steps.version.outputs.version }}" + echo "Attempt $i/$ATTEMPTS: installing agentops-accelerator==$PACKAGE_VERSION" if pip install --no-cache-dir \ - "agentops-accelerator==${{ steps.version.outputs.version }}" \ + "agentops-accelerator==$PACKAGE_VERSION" \ --index-url https://test.pypi.org/simple/ \ --extra-index-url https://pypi.org/simple/; then exit 0 @@ -225,7 +255,7 @@ jobs: sleep 30 fi done - echo "::error::agentops-accelerator==${{ steps.version.outputs.version }} was not available from TestPyPI after $ATTEMPTS attempts (~6 min)." + echo "::error::agentops-accelerator==$PACKAGE_VERSION was not available from TestPyPI after $ATTEMPTS attempts (~6 min)." exit 1 - name: Smoke test @@ -234,13 +264,15 @@ jobs: agentops --help - name: Summary + env: + PACKAGE_VERSION: ${{ needs.publish-dev.outputs.version }} run: | echo "## ✅ Dev build published and verified" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "- Version: \`${{ steps.version.outputs.version }}\`" >> "$GITHUB_STEP_SUMMARY" - echo "- TestPyPI: https://test.pypi.org/project/agentops-accelerator/${{ steps.version.outputs.version }}/" >> "$GITHUB_STEP_SUMMARY" + echo "- Version: \`$PACKAGE_VERSION\`" >> "$GITHUB_STEP_SUMMARY" + echo "- TestPyPI: https://test.pypi.org/project/agentops-accelerator/$PACKAGE_VERSION/" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Install: \`pip install agentops-accelerator==${{ steps.version.outputs.version }} --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/\`" >> "$GITHUB_STEP_SUMMARY" + echo "Install: \`pip install agentops-accelerator==$PACKAGE_VERSION --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/\`" >> "$GITHUB_STEP_SUMMARY" # Validate that the VSIX extension packages correctly build-vsix: diff --git a/tests/unit/test_ci_workflow.py b/tests/unit/test_ci_workflow.py new file mode 100644 index 0000000..a0beff9 --- /dev/null +++ b/tests/unit/test_ci_workflow.py @@ -0,0 +1,43 @@ +"""Regression tests for the repository's CI workflow.""" + +from pathlib import Path + +import yaml + + +_CI_WORKFLOW = Path(__file__).parents[2] / ".github" / "workflows" / "ci.yml" + + +def _jobs() -> dict: + workflow = yaml.safe_load(_CI_WORKFLOW.read_text(encoding="utf-8")) + return workflow["jobs"] + + +def test_publish_dev_exposes_version_from_built_wheel_metadata() -> None: + publish_dev = _jobs()["publish-dev"] + + assert publish_dev["outputs"]["version"] == ( + "${{ steps.package-version.outputs.version }}" + ) + version_step = next( + step + for step in publish_dev["steps"] + if step.get("id") == "package-version" + ) + script = version_step["run"] + assert "uv run python - <<'PY'" in script + assert 'Path("dist").glob("*.whl")' in script + assert '.endswith(".dist-info/METADATA")' in script + assert 'metadata.get("Version")' in script + assert 'echo "version=$VERSION" >> "$GITHUB_OUTPUT"' in script + + +def test_verify_dev_consumes_publish_dev_artifact_version() -> None: + verify_dev = _jobs()["verify-dev"] + serialized = yaml.safe_dump(verify_dev) + + assert verify_dev["needs"] == "publish-dev" + assert "${{ needs.publish-dev.outputs.version }}" in serialized + assert "PACKAGE_VERSION" in serialized + assert "setuptools_scm" not in serialized + assert "Determine expected version" not in serialized From 4c2ca1dd60d1e63a91b7dc876c51cc36d39aa358 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Thu, 13 Aug 2026 23:00:27 -0300 Subject: [PATCH 2/8] docs: simplify Observe guidance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/index.md | 2 +- docs/observe.md | 61 ++++++++----------------------------------------- 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/docs/index.md b/docs/index.md index b53709b..5194282 100644 --- a/docs/index.md +++ b/docs/index.md @@ -79,7 +79,7 @@ candidate versions become a release. Start with ### :material-radar: Observe [Observe](observe.md) covers Foundry traces and Azure Monitor, and how production signals feed continuous evaluation. Start with -`agentops telemetry validate` and the `agentops-agent` skill. +`agentops eval promote-traces` and the `agentops-agent` skill.
diff --git a/docs/observe.md b/docs/observe.md index 77ae26f..8055ae1 100644 --- a/docs/observe.md +++ b/docs/observe.md @@ -1,9 +1,9 @@ # Observe -This page explains how AgentOps uses agent observability. Foundry and Azure -Monitor produce the runtime signal; AgentOps reads that signal so release -readiness reflects what is actually happening in production, not just what -passed in CI. +This page explains how AgentOps turns agent observability into release evidence +and regression coverage. Foundry and Azure Monitor produce the runtime signal; +AgentOps reads that signal so readiness reflects what is actually happening in +production, not just what passed in CI. Observability is conceptual here. For the hands-on portal and KQL walkthrough, see step 18 of the [Foundry Prompt Agent tutorial](tutorial-prompt-agent.md). @@ -31,25 +31,6 @@ falls back to that connection string when discovery is not available. `agentops.agent.finding.*` spans, both of which the Cockpit can deep-link into Azure Monitor Logs. -## Operations dashboard - -Traces answer "what did this run do." Operational metrics answer "is the -deployment healthy." AgentOps ships an Azure Monitor workbook for the Foundry / -Azure OpenAI deployments behind your agent, so operators read PTU utilization, -PAYG spillover, throughput, latency percentiles, and error and throttling rates -in one place. - -The workbook is scoped per Azure OpenAI resource and per Log Analytics -workspace, with tabs for capacity, traffic and tokens, latency, and errors and -throttling. You can deploy it, open it, or export the JSON with the CLI, or -import it by hand into Azure Monitor. - -!!! info "Doctor checks the diagnostic settings" - The dashboard needs the Azure OpenAI resource to send `RequestResponse` and - `AzureOpenAIRequestUsage` logs to Log Analytics. `agentops doctor` flags - when they are missing (rule `waf.observability.aoai_diagnostic_categories`) - and prints the exact `az monitor diagnostic-settings` command to fix it. - ## Traces as evaluation signal A single trace shows what one request did. The value for release readiness comes @@ -157,43 +138,21 @@ production. ## Try it -Confirm the signal is flowing, then turn real traces into regression coverage. - -1. Check that AgentOps can reach Application Insights before you rely on the signal. +Turn reviewed production traces into regression coverage. - ```bash - agentops telemetry validate - ``` +1. Export or curate representative Foundry or Application Insights traces as + JSON or JSONL, then review them for quality and sensitive data. -2. Preview the traces and evaluation events AgentOps can currently see. - - ```bash - agentops telemetry preview - ``` - -3. Import a trace export so it can become regression coverage. - - ```bash - agentops telemetry import - ``` - -4. Promote reviewed production traces into regression dataset rows. +2. Preview how AgentOps converts the reviewed traces into regression candidates. ```bash agentops eval promote-traces --source .agentops/traces/export.jsonl ``` -5. Preview the operations dashboard as an ARM template, without touching Azure. - - ```bash - agentops telemetry dashboard deploy --dry-run - ``` - -6. Deploy the workbook, then open it in the Azure portal. +3. If the candidates are suitable, apply them to the regression dataset. ```bash - agentops telemetry dashboard deploy - agentops telemetry dashboard open + agentops eval promote-traces --source .agentops/traces/export.jsonl --apply ``` To browse this signal interactively and deep-link into Foundry and Azure From 4f367259827cfd4729c07da3e59d9d3a5fceb687 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Thu, 13 Aug 2026 23:07:12 -0300 Subject: [PATCH 3/8] docs: remove untested trace promotion guidance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/index.md | 4 ++-- docs/observe.md | 45 ++------------------------------------------- 2 files changed, 4 insertions(+), 45 deletions(-) diff --git a/docs/index.md b/docs/index.md index 5194282..c2ad57d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -78,8 +78,8 @@ candidate versions become a release. Start with
### :material-radar: Observe [Observe](observe.md) covers Foundry traces and Azure Monitor, and how -production signals feed continuous evaluation. Start with -`agentops eval promote-traces` and the `agentops-agent` skill. +production signals inform release readiness. Start with the `agentops-agent` +skill.
diff --git a/docs/observe.md b/docs/observe.md index 8055ae1..5ce8e36 100644 --- a/docs/observe.md +++ b/docs/observe.md @@ -115,54 +115,14 @@ is a governance improvement rather than a correctness gate. possible from the Azure Monitor side. It does not push traces into Agent 365. -## Trace-to-regression promotion - -The strongest use of observability is turning real production behavior into new -evaluation coverage. Reviewed production traces become new dataset rows, so the -cases your agent actually sees keep getting evaluated on every future run. - -In Foundry, this is the trace-to-dataset flow: sample recent traces, let -intelligent sampling deduplicate and select a representative set, and create an -evaluation dataset from them. AgentOps then promotes that into reviewable -regression rows with `agentops eval promote-traces`. - -!!! warning "Promotion is review-first" - Trace-derived rows are candidates, not ground truth. Self-similarity labels - are useful for drift detection, not human-verified correctness, so a person - should confirm or fill the expected answers before those rows gate a - release. This keeps regression data trustworthy as it grows. - -The loop is the point: traces become datasets, datasets gate the next release, -and the agent keeps getting evaluated on the behavior that matters in -production. - -## Try it - -Turn reviewed production traces into regression coverage. - -1. Export or curate representative Foundry or Application Insights traces as - JSON or JSONL, then review them for quality and sensitive data. - -2. Preview how AgentOps converts the reviewed traces into regression candidates. - - ```bash - agentops eval promote-traces --source .agentops/traces/export.jsonl - ``` - -3. If the candidates are suitable, apply them to the regression dataset. - - ```bash - agentops eval promote-traces --source .agentops/traces/export.jsonl --apply - ``` - To browse this signal interactively and deep-link into Foundry and Azure Monitor, run `agentops cockpit`. That local command center is covered on the [Operate](operate.md#cockpit) page. ## Run from your coding agent -Install the AgentOps skills so your coding agent can read telemetry and grow the -regression set for you. +Install the AgentOps skills so your coding agent can read telemetry and +investigate production health. ```bash agentops skills install --platform copilot @@ -173,7 +133,6 @@ The skills that map to observability are: | Skill | What it helps with | |---|---| | `agentops-agent` | Watchdog analysis of production health and latency spikes. | -| `agentops-eval` | Promote traces and re-evaluate against the hardened dataset. | ## Next From 8a7d2633333085eab85ade470401c3e74193c813 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 14 Aug 2026 00:06:28 -0300 Subject: [PATCH 4/8] docs: tighten Observe introduction Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/observe.md | 5 +---- docs/tutorial-end-to-end.md | 2 +- docs/tutorial-prompt-agent.md | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/observe.md b/docs/observe.md index 5ce8e36..c02ff2e 100644 --- a/docs/observe.md +++ b/docs/observe.md @@ -5,9 +5,6 @@ and regression coverage. Foundry and Azure Monitor produce the runtime signal; AgentOps reads that signal so readiness reflects what is actually happening in production, not just what passed in CI. -Observability is conceptual here. For the hands-on portal and KQL walkthrough, -see step 18 of the [Foundry Prompt Agent tutorial](tutorial-prompt-agent.md). - ## Where the signal comes from Foundry gives you the runtime view of an agent: traces, conversations, spans, @@ -41,7 +38,7 @@ The Doctor turns this into findings. It reads App Insights for p95 latency and error rate, and it reports when telemetry is connected but silent, so a project with no monitoring does not look healthy simply because nothing is being graded. -!!! note "Real telemetry produces honest findings" +!!! note "Real telemetry surfaces production findings" Because the Doctor reads live runtime data, it can surface latency or error findings from your own production traffic, separate from the eval gate. That is intended: a real release should investigate latency and errors before diff --git a/docs/tutorial-end-to-end.md b/docs/tutorial-end-to-end.md index 914df29..5a6e91f 100644 --- a/docs/tutorial-end-to-end.md +++ b/docs/tutorial-end-to-end.md @@ -883,7 +883,7 @@ regression candidates may not exist yet. That is useful tutorial feedback, not a failure of Doctor. If production telemetry *does* carry enough live traffic to trip latency or -error criticals, those are honest signals — not tutorial noise. The thresholds +error criticals, those are production signals — not tutorial noise. The thresholds that decide critical-vs-warning live in `.agentops/agent.yaml` (`checks.latency.p95_threshold_seconds`, `checks.errors.rate_threshold`) and are separate from the `agentops.yaml` eval-gate thresholds; raise them only if you diff --git a/docs/tutorial-prompt-agent.md b/docs/tutorial-prompt-agent.md index 89d4d03..b277f96 100644 --- a/docs/tutorial-prompt-agent.md +++ b/docs/tutorial-prompt-agent.md @@ -282,7 +282,7 @@ agents AgentOps uses judge-based quality and completeness on this shape. These few rows are the contract your agent has to keep passing. Because an LLM judge reads `expected` as acceptance criteria, you are encoding the behavior you care about, not memorizing one correct sentence. Start small and - honest: a handful of rows that capture real requirements is worth more than a + concrete: a handful of rows that capture real requirements is worth more than a large set that nobody trusts. ```text @@ -453,7 +453,7 @@ server-side evaluator setup. Confirm the eval runner the generator will use: !!! concept "What the smoke gate proves" - A smoke gate is the smallest honest test: a few rows, run end to end against + A smoke gate is the smallest meaningful test: a few rows, run end to end against the real agent, scored by a judge. It will not catch every regression, and it is not meant to. Its job is to fail fast and loud when something is obviously broken, so you trust green to mean "safe to keep going." You harden it into a From 53e34850475fd7e86820b09d8f9fd44d826fed6e Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 14 Aug 2026 00:25:21 -0300 Subject: [PATCH 5/8] docs: explain OpenTelemetry agent tracing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/observe.md | 122 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/docs/observe.md b/docs/observe.md index c02ff2e..0d10281 100644 --- a/docs/observe.md +++ b/docs/observe.md @@ -23,11 +23,131 @@ AgentOps first tries to auto-discover the project's App Insights resource and falls back to that connection string when discovery is not available. !!! info "Telemetry from CI runs" - Generated eval and Doctor workflows install AgentOps telemetry support. + Generated eval and Doctor workflows install OpenTelemetry support. Eval runs emit `agentops.eval.*` spans and scheduled Doctor runs emit `agentops.agent.finding.*` spans, both of which the Cockpit can deep-link into Azure Monitor Logs. +## OpenTelemetry spans and semantic conventions + +An OpenTelemetry trace is a tree of spans. Each span represents one unit of +work and records a name, start and end timestamps, status, attributes, and its +parent/child relationship to other spans. The OpenTelemetry generative AI +semantic conventions give those spans consistent names and `gen_ai.*` +attributes, so the same telemetry remains meaningful in Foundry and Azure +Monitor. + +| Work | Span and operation | Key attributes | +|---|---|---| +| Agent invocation | `invoke_agent `; `gen_ai.operation.name=invoke_agent` | `gen_ai.agent.name`, `gen_ai.agent.id`, `gen_ai.conversation.id` | +| Model call | `chat `; `gen_ai.operation.name=chat` | `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | +| Tool execution | `execute_tool `; `gen_ai.operation.name=execute_tool` | `gen_ai.tool.name` | + +Domain attributes such as `helpdesk.ticket.queue` can add business context, but +they complement rather than replace the standard `gen_ai.*` attributes. + +!!! warning "Keep sensitive content out of telemetry" + Do not record secrets, tokens, or personal data in span attributes. + `gen_ai.input.messages`, `gen_ai.output.messages`, and tool arguments or + results can contain sensitive content, so omit, minimize, or redact them. + +## Configure an agent to emit telemetry + +Agents hosted in Foundry receive server-side tracing after you connect an +Application Insights resource to the project. If you own application code +around the agent call, add client-side instrumentation to capture that custom +logic as well. + +Install the Azure Monitor OpenTelemetry distribution: + +```bash +pip install azure-monitor-opentelemetry +``` + +Set `APPLICATIONINSIGHTS_CONNECTION_STRING` in the process environment; using +an application setting or secret reference is recommended in production. Then +configure Azure Monitor once during application startup: + +```python +from azure.monitor.opentelemetry import configure_azure_monitor +from opentelemetry.sdk.resources import Resource + +configure_azure_monitor( + resource=Resource.create({"service.name": "helpdesk-agent"}), +) +``` + +Do not hardcode the connection string in source. After startup, run the agent +and verify a new trace in **Foundry > Agents > Traces** or **Application +Insights > Investigate > Agents (Preview)**. See the official +[Foundry tracing setup](https://learn.microsoft.com/azure/foundry/observability/how-to/trace-agent-setup) +for the complete connection, permissions, and verification flow. + +## Instrument a custom Python agent + +Once Azure Monitor is configured, application code can reuse the global tracer +provider without depending on an agent framework: + +```python +from opentelemetry import trace +from opentelemetry.trace import Status, StatusCode + +tracer = trace.get_tracer("contoso.helpdesk.agent") + + +def lookup_ticket(ticket_id: str) -> str: + return f"Ticket {ticket_id} is queued for review." + + +def run_agent(ticket_id: str, conversation_id: str) -> str: + agent_attributes = { + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "helpdesk-agent", + "gen_ai.agent.id": "helpdesk-agent-v1", + "gen_ai.conversation.id": conversation_id, + } + with tracer.start_as_current_span( + "invoke_agent helpdesk-agent", + attributes=agent_attributes, + record_exception=False, + set_status_on_exception=False, + ) as agent_span: + try: + with tracer.start_as_current_span( + "execute_tool lookup_ticket", + attributes={ + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "lookup_ticket", + }, + record_exception=False, + set_status_on_exception=False, + ) as tool_span: + try: + ticket = lookup_ticket(ticket_id) + except Exception as exc: + tool_span.set_attribute("error.type", type(exc).__name__) + tool_span.set_status(Status(StatusCode.ERROR)) + raise + + agent_span.set_status(Status(StatusCode.OK)) + return f"Helpdesk response: {ticket}" + except Exception as exc: + agent_span.set_attribute("error.type", type(exc).__name__) + agent_span.set_status(Status(StatusCode.ERROR)) + raise +``` + +The span context managers preserve the parent/child relationship and finish +each span with its measured duration automatically. The example disables +automatic exception recording and records only `error.type`, avoiding exception +messages, stack traces, and tool arguments in telemetry. + +For a more specific policy-span implementation, see the +[`acs_middleware.py` example](https://github.com/placerda/safe-agent-on-foundry/blob/main/src/helpdeskbot/acs_middleware.py#L280). +The official guide to +[trace structure and semantic conventions](https://learn.microsoft.com/azure/foundry/how-to/develop/langchain-traces#understand-trace-structure) +describes the same agent, model, and tool span hierarchy in detail. + ## Traces as evaluation signal A single trace shows what one request did. The value for release readiness comes From f4523511a197896af0fb9d57b45d3486becb3840 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 14 Aug 2026 06:21:44 -0300 Subject: [PATCH 6/8] docs: remove untested agent identity guidance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/observe.md | 72 +------------------------------------------------ 1 file changed, 1 insertion(+), 71 deletions(-) diff --git a/docs/observe.md b/docs/observe.md index 0d10281..36ca320 100644 --- a/docs/observe.md +++ b/docs/observe.md @@ -164,77 +164,7 @@ with no monitoring does not look healthy simply because nothing is being graded. is intended: a real release should investigate latency and errors before promoting, even when the candidate's eval scores pass. -## Agent identity on traces - -Traces tell you what an agent did. They do not, by default, tell you *which* -agent did it in a way an auditor can reconcile with your tenant. Microsoft -Entra Agent ID closes that gap: the agent gets a first-class identity, and the -same identifier travels from registration through traces into release evidence. - -The handshake has three steps, and each one is a different tool, so it is worth -being explicit about who writes what. - -**1. Register the identity.** `agentops agent register` creates (or adopts) an -agent identity blueprint in Microsoft Entra and records the resolved -application id locally: - -```bash -agentops agent register --sponsor owner@contoso.com -``` - -The sponsor is required. An agent identity with no accountable owner cannot be -governed, so there is no default. The command is idempotent: if a blueprint -with the same display name already exists, AgentOps reuses it instead of -creating a duplicate. Run it with `--dry-run` first to see the resolved display -name and sponsor without calling Microsoft Graph. - -The resolved id is written to `.agentops/identity/agent-identity.json`. Declare -the inputs in `agentops.yaml` so they are source-controlled: - -```yaml -identity: - display_name: support-agent - sponsor: owner@contoso.com - verify: true -``` - -`verify: true` tells the Doctor to confirm the blueprint against Microsoft -Graph. It is off by default because that lookup needs tenant admin consent -(`AgentIdentityBlueprint.Read.All`), which most workspaces will not have on day -one. With it off, the Doctor still reports whether an identity is registered at -all, using only local state. - -**2. Stamp it on traces.** Once an identity is resolved, AgentOps adds it to -the OpenTelemetry resource as `gen_ai.agent.id`, so every span AgentOps emits -carries the Entra Agent ID. In CI, where the local record is not checked in, -set `AGENTOPS_ENTRA_AGENT_ID` instead and the attribute resolves from the -environment. - -The attribute is **omitted** when no identity is registered, never emitted as an -empty string. That distinction matters when you query: filtering on presence -tells you which traffic is attributable and which is not. - -```kusto -dependencies -| where isnotempty(customDimensions["gen_ai.agent.id"]) -| summarize runs = count() by tostring(customDimensions["gen_ai.agent.id"]) -``` - -**3. Publish it as evidence.** The release evidence pack reads the same record -and adds an `agent_identity` section reporting the id and where it came from -(the local record or the environment variable). When no identity is registered, -the pack raises a warning rather than a blocker, because identity registration -is a governance improvement rather than a correctness gate. - -!!! note "AgentOps does not ingest into Agent 365" - There is no public ingestion API for Agent 365 telemetry today. AgentOps - stamps the identifier and publishes it as evidence so the correlation is - possible from the Azure Monitor side. It does not push traces into Agent - 365. - -To browse this signal interactively and deep-link into Foundry and Azure -Monitor, run `agentops cockpit`. That local command center is covered on the -[Operate](operate.md#cockpit) page. + ## Run from your coding agent From 23e028334895bce7909049484089555f1a0874c3 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 14 Aug 2026 06:23:27 -0300 Subject: [PATCH 7/8] feat: simplify AgentOps observability Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7a36e0b5-1c09-4953-8e8e-3f8aa2e1f1a9 --- CHANGELOG.md | 31 + pyproject.toml | 3 - src/agentops/agent/checks/catalog.py | 13 - .../agent/checks/posture_rules/__init__.py | 4 - .../aoai_diagnostic_categories.py | 88 -- src/agentops/agent/cockpit.py | 1164 +++++------------ src/agentops/agent/sources/azure_monitor.py | 34 +- src/agentops/cli/app.py | 385 ------ src/agentops/core/agentops_config.py | 119 -- src/agentops/services/agent_identity.py | 11 +- src/agentops/services/dashboard.py | 506 ------- src/agentops/services/telemetry_import.py | 550 -------- src/agentops/templates/workbooks/README.md | 169 --- .../workbooks/foundry-ops.workbook.json | 683 ---------- .../workbooks/queries/agent_behavior.kql | 222 ---- .../queries/capacity_ptu_spillover.kql | 52 - .../workbooks/queries/errors_throttling.kql | 48 - .../workbooks/queries/latency_percentiles.kql | 74 -- .../workbooks/queries/traffic_tokens.kql | 61 - src/agentops/utils/foundry_discovery.py | 85 ++ tests/unit/test_agent_identity_service.py | 10 + tests/unit/test_agent_posture_rules.py | 70 - tests/unit/test_agentops_config.py | 104 -- tests/unit/test_cli_commands.py | 108 +- tests/unit/test_cli_dashboard.py | 156 --- tests/unit/test_cockpit.py | 687 ++++------ tests/unit/test_dashboard.py | 509 ------- tests/unit/test_foundry_discovery.py | 33 + tests/unit/test_telemetry.py | 56 + tests/unit/test_telemetry_import.py | 153 --- 30 files changed, 818 insertions(+), 5370 deletions(-) delete mode 100644 src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py delete mode 100644 src/agentops/services/dashboard.py delete mode 100644 src/agentops/services/telemetry_import.py delete mode 100644 src/agentops/templates/workbooks/README.md delete mode 100644 src/agentops/templates/workbooks/foundry-ops.workbook.json delete mode 100644 src/agentops/templates/workbooks/queries/agent_behavior.kql delete mode 100644 src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql delete mode 100644 src/agentops/templates/workbooks/queries/errors_throttling.kql delete mode 100644 src/agentops/templates/workbooks/queries/latency_percentiles.kql delete mode 100644 src/agentops/templates/workbooks/queries/traffic_tokens.kql delete mode 100644 tests/unit/test_cli_dashboard.py delete mode 100644 tests/unit/test_dashboard.py delete mode 100644 tests/unit/test_telemetry_import.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 00de98c..eeac465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Changed +- **The AgentOps Cockpit now focuses on five release-readiness sections.** + The page renders only the readiness and Doctor ship verdicts, Foundry and + GitHub connections, the 13-item observability checklist, the latest Doctor + findings, and prioritized next actions. Date-window and auto-refresh controls + were removed because the remaining sections describe current configuration + and the latest analysis rather than time-series dashboards. +- **Observability readiness now distinguishes native Foundry tracing from + optional custom spans.** Hosted agents and prompt agents are recognized as + natively instrumented, while repository OpenTelemetry spans are reported as + optional extensions. App Insights linkage, rubric evaluators, alert + definitions, and tracing evidence are resolved independently so unknown state + is no longer presented as a failure. + +### Fixed +- **Project-managed Application Insights connections now work throughout + Doctor and Cockpit.** AgentOps discovers the attached Application Insights + ARM resource from credential-free Foundry connection metadata and queries it + with `LogsQueryClient.query_resource`, avoiding false missing-telemetry + findings when no API-key connection string exists. +- **`agentops agent register` now derives hosted-agent names from Foundry + target URLs.** A command that supplies only `--sponsor` correctly resolves + names such as `helpdeskbot`, matching the fallback promised by `--help`. + +### Removed +- **The experimental AgentOps telemetry import and custom Operations Dashboard + have been removed.** The public telemetry command group, Log Analytics + workbook deployment, bundled workbook/KQL assets, and dashboard-specific + posture rule are no longer shipped while native product observability support + is reviewed. + ## [0.8.8] - 2026-08-14 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 5682a5e..123a73d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,9 +64,6 @@ where = ["src"] "pipelines/azuredevops/*.yml", "skills/*/SKILL.md", "agent-server/*", - "workbooks/*.json", - "workbooks/*.md", - "workbooks/queries/*.kql", ] "agentops.agent.knowledge" = ["*.csv"] diff --git a/src/agentops/agent/checks/catalog.py b/src/agentops/agent/checks/catalog.py index b25a22a..05b0baf 100644 --- a/src/agentops/agent/checks/catalog.py +++ b/src/agentops/agent/checks/catalog.py @@ -775,19 +775,6 @@ def is_llm_judged(self) -> bool: severities=(Severity.WARNING, Severity.CRITICAL), requires=("azure_resources",), ), - CheckSpec( - id="waf.observability.aoai_diagnostic_categories", - category=Category.OPERATIONAL_EXCELLENCE, - title="Azure OpenAI usage telemetry categories are not enabled", - summary=( - "The Azure OpenAI account is not emitting the RequestResponse " - "and AzureOpenAIRequestUsage diagnostic log categories to a Log " - "Analytics workspace, so the Foundry operations dashboard and " - "any token / latency / throttling analysis render empty." - ), - severities=(Severity.WARNING,), - requires=("azure_resources",), - ), # ------------------------------------------------------------------ # Responsible AI # ------------------------------------------------------------------ diff --git a/src/agentops/agent/checks/posture_rules/__init__.py b/src/agentops/agent/checks/posture_rules/__init__.py index 109989a..fb25927 100644 --- a/src/agentops/agent/checks/posture_rules/__init__.py +++ b/src/agentops/agent/checks/posture_rules/__init__.py @@ -36,9 +36,6 @@ def _build_registry() -> Dict[str, RuleFn]: from agentops.agent.checks.posture_rules.diagnostics import ( evaluate as diagnostics_rule, ) - from agentops.agent.checks.posture_rules.aoai_diagnostic_categories import ( - evaluate as aoai_diagnostic_categories_rule, - ) from agentops.agent.checks.posture_rules.local_auth import ( evaluate as local_auth_rule, ) @@ -50,7 +47,6 @@ def _build_registry() -> Dict[str, RuleFn]: "waf.security.local_auth_disabled": local_auth_rule, "waf.security.managed_identity": managed_identity_rule, "waf.security.diagnostic_settings": diagnostics_rule, - "waf.observability.aoai_diagnostic_categories": aoai_diagnostic_categories_rule, } diff --git a/src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py b/src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py deleted file mode 100644 index aa97b05..0000000 --- a/src/agentops/agent/checks/posture_rules/aoai_diagnostic_categories.py +++ /dev/null @@ -1,88 +0,0 @@ -"""WAF-AI Operational Excellence: Azure OpenAI usage telemetry must flow. - -The Foundry operations dashboard (``agentops telemetry dashboard``) and any -token / latency / throttling analysis depend on two diagnostic log categories -being enabled on the Azure OpenAI (Cognitive Services) account and routed to a -Log Analytics workspace: - -* ``RequestResponse`` - per-request traces (status codes, streaming). -* ``AzureOpenAIRequestUsage`` - prompt / generated token counts. - -When either category is missing the workbook tiles render empty. This rule -fires when the account does not emit both categories and prints the exact -``az monitor diagnostic-settings create`` command to fix it. The check is -read-only; it never changes Azure. -""" - -from __future__ import annotations - -import json -from typing import List - -from agentops.agent.findings import Category, Finding, Severity -from agentops.agent.sources.azure_resources import AzureResourcesPayload - -RULE_ID = "waf.observability.aoai_diagnostic_categories" - -REQUIRED_CATEGORIES = ("RequestResponse", "AzureOpenAIRequestUsage") - - -def _fix_command(account_id: str, workspace_id: str) -> str: - logs = json.dumps([{"category": c, "enabled": True} for c in REQUIRED_CATEGORIES]) - return ( - "az monitor diagnostic-settings create " - "--name agentops-foundry-ops " - f"--resource {account_id} " - f"--workspace {workspace_id} " - f"--logs '{logs}'" - ) - - -def evaluate(payload: AzureResourcesPayload, source_name: str) -> List[Finding]: - account = payload.account - if account is None: - return [] - - enabled: set[str] = set() - for setting in payload.diagnostic_settings: - for category in setting.enabled_log_categories: - enabled.add(str(category)) - - missing = [c for c in REQUIRED_CATEGORIES if c not in enabled] - if not missing: - return [] - - account_id = getattr(account, "id", None) or f"<{account.name}-resource-id>" - workspace_id = next( - (s.workspace_id for s in payload.diagnostic_settings if s.workspace_id), - "", - ) - - return [ - Finding( - id=RULE_ID, - severity=Severity.WARNING, - category=Category.OPERATIONAL_EXCELLENCE, - title="Azure OpenAI usage telemetry categories are not enabled", - summary=( - f"Azure OpenAI account `{account.name}` is not emitting the " - f"`{'`, `'.join(missing)}` diagnostic log " - f"{'category' if len(missing) == 1 else 'categories'}. The " - "Foundry operations dashboard and any token, latency, or " - "throttling analysis need both `RequestResponse` and " - "`AzureOpenAIRequestUsage` streamed to a Log Analytics " - "workspace, so those tiles will render empty." - ), - recommendation=( - "Enable the missing categories with:\n" - f"{_fix_command(account_id, workspace_id)}" - ), - source=source_name, - evidence={ - "account": account.name, - "required_categories": list(REQUIRED_CATEGORIES), - "missing_categories": missing, - "enabled_categories": sorted(enabled), - }, - ) - ] diff --git a/src/agentops/agent/cockpit.py b/src/agentops/agent/cockpit.py index cfd07c9..8f5a0b6 100644 --- a/src/agentops/agent/cockpit.py +++ b/src/agentops/agent/cockpit.py @@ -26,7 +26,7 @@ from urllib.parse import quote from agentops.agent.history import AnalysisRecord, load_analysis_history -from agentops.agent.time_range import TimeRange, parse_time_range, preset_keys +from agentops.agent.time_range import TimeRange from agentops.utils.yaml import load_yaml @@ -70,65 +70,25 @@ def build_cockpit_payload( history: Optional[List[AnalysisRecord]] = None, time_range: Optional[TimeRange] = None, ) -> Dict[str, Any]: - """Reduce raw history + eval runs into a cockpit-ready dict. - - Note: the production section is **not** fetched here. It is rendered - as a placeholder in the initial HTML and filled in asynchronously by - the browser hitting ``/api/production/html``. This keeps the initial - page load fast (local file reads only) even when App Insights is - slow to authenticate or query. - """ - if time_range is None: - time_range = parse_time_range() - all_records = history if history is not None else load_analysis_history(workspace) - records = _filter_records(all_records, time_range) - eval_runs_all = _load_eval_runs(workspace, limit=200) - eval_runs = _filter_eval_runs(eval_runs_all, time_range) + """Reduce local configuration and the latest Doctor run for the cockpit.""" + _ = time_range # Retained for callers; the cockpit no longer filters by date. + records = history if history is not None else load_analysis_history(workspace) telemetry = _telemetry_status() - # Production is deferred to /api/production/html; render a placeholder. - production = {"has_data": False, "deferred": telemetry.get("enabled", False), "cards": []} - - eval_payload = _build_eval_section(eval_runs) - eval_payload["official_eval"] = _official_eval_artifact_status(workspace) watchdog_payload = _build_watchdog_section(records) - deployments_payload = _build_deployments_section(workspace, time_range) - foundry_connection = _build_foundry_connection(workspace, telemetry) - open_in_foundry = _build_open_in_foundry(workspace, telemetry) readiness = _build_readiness_checklist( - workspace, telemetry, deployments_payload, watchdog_payload, + workspace, telemetry, watchdog=watchdog_payload, ) next_actions = _build_next_actions( - workspace, telemetry, watchdog_payload, readiness, eval_payload, + watchdog_payload, readiness, ) return { "workspace": str(workspace.resolve()), - "foundry_project_url": _resolve_foundry_project_url(workspace), - "foundry_compliance_url": _resolve_foundry_compliance_url(workspace), - "foundry_setup_url": _foundry_setup_url(), - "az_tenant_id": _az_tenant_id(), - "time_range": { - "key": time_range.key, - "label": time_range.label, - "start": time_range.start.isoformat(), - "end": time_range.end.isoformat(), - "hours": time_range.hours, - "query": time_range.to_query(), - }, "telemetry": telemetry, - "production": production, - "eval": eval_payload, - "metrics": _build_metrics_cards(eval_runs), "watchdog": watchdog_payload, - "deployments": deployments_payload, - "foundry_connection": foundry_connection, - "open_in_foundry": open_in_foundry, + "connections": _build_connections(workspace), "readiness": readiness, "next_actions": next_actions, - "summary_counts": { - "eval_runs": len(eval_runs), - "analyses": len(records), - }, } @@ -1488,13 +1448,19 @@ def _telemetry_status() -> Dict[str, Any]: } if project: reason: Optional[str] = None + resource_reason: Optional[str] = None try: from agentops.utils.foundry_discovery import ( resolve_appinsights_connection_from_env_with_reason, + resolve_appinsights_resource_id_from_env_with_reason, ) conn, reason = resolve_appinsights_connection_from_env_with_reason() + resource_id, resource_reason = ( + resolve_appinsights_resource_id_from_env_with_reason() + ) except Exception as exc: # noqa: BLE001 conn = None + resource_id = None reason = f"discovery raised {type(exc).__name__}: {exc}" if conn: portal_url = _appinsights_portal_url(conn) @@ -1508,13 +1474,35 @@ def _telemetry_status() -> Dict[str, Any]: "doctor_findings_url": _appinsights_doctor_findings_portal_url(conn), "tone": "ok", } + if resource_id: + return { + "enabled": True, + "source": "foundry_project_connection", + "label": "App Insights", + "detail": ( + "Linked to the Foundry project with Project Managed Identity." + ), + "hint": ( + "Resolved from credential-free Foundry connection metadata; " + "an API key connection string is not required." + ), + "resource_id": resource_id, + "portal_url": _azure_resource_portal_url(resource_id), + "tone": "ok", + } # Surface the actual reason inline so the user does not have to # tail the cockpit server logs to learn why discovery failed. + connection_is_pmi = bool(reason and "ProjectManagedIdentity" in reason) + failure_reason = ( + resource_reason or reason + if connection_is_pmi + else reason or resource_reason + ) reason_html = ( f'
' - f'Why: {_html_escape(reason)}' + f'Why: {_html_escape(failure_reason)}' "
" - if reason + if failure_reason else "" ) return { @@ -1588,6 +1576,11 @@ def _appinsights_portal_url(connection_string: Optional[str]) -> Optional[str]: return _appinsights_logs_url(app_id, query) +def _azure_resource_portal_url(resource_id: str) -> str: + """Build a portal link without requiring App Insights API-key metadata.""" + return f"https://portal.azure.com/#resource{resource_id}/overview" + + def _appinsights_doctor_findings_portal_url(connection_string: Optional[str]) -> Optional[str]: """Build a Logs blade link focused on AgentOps Doctor finding spans.""" if not connection_string: @@ -1668,24 +1661,12 @@ def _url_quote(text: str) -> str: # --------------------------------------------------------------------------- -def _build_foundry_connection( - workspace: Path, - telemetry: Dict[str, Any], -) -> Dict[str, Any]: - """Summarize how this repo connects to Microsoft Foundry. - - Inputs are read-only: env vars, run.yaml/agent.yaml, and the most - recent ``cloud_evaluation.json`` (for the project root). Cockpit - renders this as the first card on the page so users can verify they - are pointed at the right Foundry tenant/project before drilling in. - """ +def _build_connections(workspace: Path) -> Dict[str, Any]: + """Describe the Foundry project and GitHub repository in scope.""" project_env = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") - tenant = _az_tenant_id() project_url = _resolve_foundry_project_url(workspace) project_root = _resolve_foundry_project_root(workspace) - agent_id, agent_source = _resolve_agent_identity(workspace) - if project_env: project_status = "ok" project_label = "Project endpoint configured" @@ -1705,40 +1686,9 @@ def _build_foundry_connection( ) project_copy_value = None - if tenant: - tenant_status = "ok" - tenant_label = "Azure tenant resolved" - tenant_detail = f"{_html_escape(tenant)}" - tenant_hint = "Resolved from `az account show`." - else: - tenant_status = "warn" - tenant_label = "Azure tenant unknown" - tenant_detail = ( - "Run az login so Foundry deep-links open in " - "the correct directory." - ) - tenant_hint = None - - if agent_id: - agent_status = "ok" - agent_label = "Agent configured" - agent_detail = ( - f"{_html_escape(agent_id)}" - ) - agent_hint = f"Resolved from {agent_source}." - else: - agent_status = "muted" - agent_label = "No agent pinned" - agent_detail = ( - "Set agent in agentops.yaml when " - "you want the cockpit to surface a specific Foundry agent." - ) - agent_hint = None - - telemetry_status = telemetry.get("tone", "muted") - telemetry_label = telemetry.get("label", "Telemetry off") - telemetry_detail = telemetry.get("detail", "") - telemetry_hint = telemetry.get("hint") + github = _resolve_github_repository(workspace) + github_name = github.get("name") + github_url = github.get("url") items = [ { @@ -1751,34 +1701,46 @@ def _build_foundry_connection( "copy_value": project_copy_value, }, { - "title": "Azure tenant", - "status": tenant_status, - "label": tenant_label, - "detail": tenant_detail, - "hint": tenant_hint, - }, - { - "title": "Agent", - "status": agent_status, - "label": agent_label, - "detail": agent_detail, - "hint": agent_hint, - "link": (_foundry_deeplinks(workspace).get("agent") if agent_id else None), - "link_label": "Open agent", - }, - { - "title": "Application Insights", - "status": telemetry_status, - "label": telemetry_label, - "detail": telemetry_detail, - "hint": telemetry_hint, - "link": telemetry.get("portal_url"), - "link_label": "Open App Insights", + "title": "GitHub repository", + "status": "ok" if github_url else "warn", + "label": github_name or "GitHub repository missing", + "detail": ( + "Repository remote resolved from local Git configuration." + if github_url + else "Configure an origin remote that points to GitHub." + ), + "link": github_url, + "link_label": "Open in GitHub", + "copy_value": github_url, }, ] + return {"items": items} + + +def _resolve_github_repository(workspace: Path) -> Dict[str, Optional[str]]: + """Resolve a browser URL from the local ``origin`` remote.""" + proc = _run_quick(["git", "remote", "get-url", "origin"], cwd=workspace) + if proc is None or proc.returncode != 0: + return {"name": None, "url": None} + remote = (proc.stdout or "").strip() + if not remote: + return {"name": None, "url": None} + + match = re.match( + r"^(?:https?://|ssh://git@)(?P[^/:]+)[/:](?P.+?)(?:\.git)?$", + remote, + ) + if not match: + match = re.match(r"^git@(?P[^:]+):(?P.+?)(?:\.git)?$", remote) + if not match or "github" not in match.group("host").lower(): + return {"name": None, "url": None} + + path = match.group("path").removesuffix(".git").strip("/") + if path.count("/") != 1: + return {"name": None, "url": None} return { - "items": items, - "has_project": bool(project_env or project_root), + "name": path, + "url": f"https://{match.group('host')}/{path}", } @@ -1791,14 +1753,13 @@ def _build_open_in_foundry( Cockpit surfaces a curated panel of Foundry links so users can drill down without manually navigating the portal. Azure Monitor surfaces - (raw App Insights telemetry and the Foundry operations workbook) are - folded into the Foundry project subgroup so there is a single place to - look, rather than a duplicated one-tile group. + (raw App Insights telemetry) are folded into the Foundry project subgroup + so there is a single place to look, rather than a duplicated one-tile + group. """ deeplinks = _foundry_deeplinks(workspace) portal_url = telemetry.get("portal_url") if isinstance(telemetry, dict) else None project_url = _resolve_foundry_project_url(workspace) - workbook_url = _resolve_workbook_portal_url(workspace) agent_targets: List[Dict[str, Any]] = [ { @@ -1851,17 +1812,6 @@ def _build_open_in_foundry( ), "url": deeplinks.get("operate") or project_url, }, - { - "key": "foundry_ops_dashboard", - "title": "Foundry operations dashboard", - "description": ( - "Azure Monitor workbook with PTU capacity, token traffic, " - "latency percentiles, throttling, and read-only Foundry " - "trace-evaluation behavior. Deploy or open it with " - "`agentops telemetry dashboard`." - ), - "url": workbook_url, - }, { "key": "app_insights", "title": "App Insights", @@ -1891,31 +1841,10 @@ def _build_open_in_foundry( } -def _resolve_workbook_portal_url(workspace: Path) -> Optional[str]: - """Return the Foundry operations workbook portal URL (read-only). - - Mirrors ``agentops telemetry dashboard open``: discovery reads the azd - ``.env`` (a file read, no Azure calls) so the deep link matches. Any - failure falls back to ``None`` so the cockpit never breaks. - """ - try: - from agentops.services import dashboard as dash - - target = dash.discover_target(workspace) - return dash.build_workbook_portal_url( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - name=target.name, - tenant_id=target.tenant_id, - ) - except Exception: # noqa: BLE001 - deep link is best-effort, cockpit stays up - return None - - def _build_readiness_checklist( workspace: Path, telemetry: Dict[str, Any], - deployments: Dict[str, Any], + deployments: Optional[Dict[str, Any]] = None, watchdog: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Read-only checklist of repo-side observability readiness. @@ -1927,6 +1856,7 @@ def _build_readiness_checklist( lives in the Monitor / Evaluations panels users open from the deep-links panel. """ + _ = deployments # Backwards-compatible input; readiness is repo/Doctor based. checks: List[Dict[str, Any]] = [] agentops_config = _read_agentops_config(workspace) trace_manifest = _read_trace_regression_manifest(workspace) @@ -1934,15 +1864,17 @@ def _build_readiness_checklist( trace_lineage: Dict[str, Any] = ( raw_trace_lineage if isinstance(raw_trace_lineage, dict) else {} ) + custom_tracing_path = _detect_custom_tracing(workspace) + foundry_runtime = _detect_foundry_agent_runtime(workspace, agentops_config) - tracing_ok = bool(telemetry.get("enabled")) + tracing_linked = bool(telemetry.get("enabled")) checks.append( { - "title": "Server-side tracing (agent → App Insights)", - "status": "ok" if tracing_ok else "warn", + "title": "App Insights connection", + "status": "ok" if tracing_linked else "warn", "detail": ( - telemetry.get("detail", "") - if tracing_ok + str(telemetry.get("detail") or "Linked to Application Insights.") + if tracing_linked else "How to complete: wire " "APPLICATIONINSIGHTS_CONNECTION_STRING or attach " "App Insights to the Foundry project. " @@ -1952,25 +1884,31 @@ def _build_readiness_checklist( } ) - # Client-side tracing is not auto-detectable from outside the - # caller process — it depends on whether the application itself - # imports the OTel SDK and instruments outbound model / agent - # calls. Surface it as an "info" reminder + docs link so the - # readiness panel stays honest about what AgentOps can verify. - client_side_ok = bool(os.getenv("AGENTOPS_CLIENT_TRACING")) or tracing_ok + agent_tracing_ready = bool(foundry_runtime or custom_tracing_path) checks.append( { - "title": "Client-side tracing (app code instrumented)", - "status": "info" if client_side_ok else "muted", + "title": "Agent tracing instrumentation", + "status": "ok" if agent_tracing_ready else "muted", "detail": ( - "How to complete: instrument the application " - "that calls your agent, not just AgentOps. Add Azure Monitor " - "OpenTelemetry to the app process, configure the same " - "APPLICATIONINSIGHTS_CONNECTION_STRING, and wrap " - "outbound model/agent/tool calls so a user request shows one " - "end-to-end trace across client code, agent execution, and " - "dependencies. Set AGENTOPS_CLIENT_TRACING=1 only " - "after the app is instrumented. " + f"Microsoft Foundry provides native tracing for this " + f"{_html_escape(foundry_runtime)}; no application-side " + "OpenTelemetry setup is required." + + ( + " Additional custom spans were detected in " + f"{_html_escape(custom_tracing_path)}." + if custom_tracing_path + else " Custom spans remain optional." + ) + if foundry_runtime + else ( + "Detected custom OpenTelemetry instrumentation in " + f"{_html_escape(custom_tracing_path)}." + ) + if custom_tracing_path + else "How to complete: instrument the agent " + "runtime with OpenTelemetry. Foundry hosted " + "agents and prompt agents provide this natively; custom or " + "external runtimes must configure their own tracer and exporter. " 'Foundry tracing docs ↗' ), @@ -2011,16 +1949,19 @@ def _build_readiness_checklist( } ) - rubrics = agentops_config.get("rubrics") - rubric_ready = isinstance(rubrics, list) and bool(rubrics) + rubric_ready, rubric_source = _detect_rubric_evaluator( + workspace, + agentops_config, + ) checks.append( { "title": "Optional rubric evaluator gate", "status": "ok" if rubric_ready else "muted", "detail": ( - "Detected rubrics: in agentops.yaml. " - "Keep thresholds bound only to metric names emitted by your " - "Foundry / azd run." + f"Detected a rubric evaluator in " + f"{_html_escape(rubric_source)}. " + "Keep thresholds bound only to metric names emitted by the " + "corresponding Foundry / azd evaluation run." if rubric_ready else "How to complete: optional - add " "rubrics: only after a real Foundry rubric evaluator " @@ -2189,29 +2130,22 @@ def _build_readiness_checklist( } ) - alerts = bool(telemetry.get("portal_url")) and tracing_ok + alerts, alerts_source = _detect_alert_configuration( + workspace, + agentops_config, + ) checks.append( { "title": "Alerts wired", - "status": "info" if alerts else "muted", + "status": "ok" if alerts else "info", "detail": ( - "App Insights is linked. Next, create Azure Monitor alert " - "rules for the agent workload: failures in " - "requests, slow P95 duration, high dependency " - "error rate, and optionally AgentOps CI/Doctor spans such as " - "agentops.eval.* and " - "agentops.agent.finding.*. " - 'Alert docs ↗' + "Detected Azure Monitor alert configuration in " + f"{_html_escape(alerts_source)}." if alerts - else "How to complete: once tracing is wired, " - "create Azure Monitor / App Insights alert rules for the " - "agent workload. Start with requests | where success == false " - "for failures, P95 request duration for latency, and " - "dependencies failures for downstream services. " - "If you want AgentOps signals in alerts too, add rules over " - "agentops.eval.* and " - "agentops.agent.finding.* custom dimensions. " + else "Not verified: Cockpit found no alert definition in repo " + "configuration, and the latest Doctor analysis does not inventory " + "Azure Monitor alert rules. This does not claim that cloud-side " + "alerts are absent. Define alerts as IaC or verify them in Azure Monitor. " 'Alert docs ↗' ), @@ -2296,87 +2230,39 @@ def _continuous_eval_status_from_watchdog( def _build_next_actions( - workspace: Path, - telemetry: Dict[str, Any], watchdog: Dict[str, Any], readiness: Dict[str, Any], - eval_payload: Dict[str, Any], ) -> Dict[str, Any]: - """Surface a short, ordered list of contextual next actions. - - Each action either opens a Cockpit section or links to the relevant - Foundry / Azure runtime view. - """ + """Prioritize Doctor findings, then incomplete readiness checks.""" actions: List[Dict[str, Any]] = [] - - if not telemetry.get("enabled"): - actions.append( - { - "title": "Wire App Insights to Foundry", - "detail": ( - "Tracing is off. Without it Foundry Monitor, " - "Evaluations, and Traces stay empty." - ), - "cta": "Open Foundry connected resources", - "url": _resolve_foundry_project_url(workspace), - } - ) - - readiness_checks = readiness.get("checks", []) - if any(c["status"] == "warn" for c in readiness_checks if c["title"].startswith("CI eval gate")): - actions.append( - { - "title": "Add a CI eval workflow", - "detail": ( - "Generate a PR workflow. Prompt-agent repos use the " - "official Microsoft eval runner when compatible; hosted " - "and fallback cases use agentops eval run." - ), - "cta": "agentops workflow generate", - } - ) - - latest_findings = watchdog.get("latest_findings") or [] - crit_findings = [f for f in latest_findings if (f.get("severity") or "").lower() == "critical"] - if crit_findings: + severity_order = {"critical": 0, "warning": 1, "info": 2} + findings = sorted( + watchdog.get("latest_findings") or [], + key=lambda item: severity_order.get(str(item.get("severity") or "").lower(), 3), + ) + for finding in findings: + recommendation = str(finding.get("recommendation") or finding.get("summary") or "") actions.append( { - "title": f"Fix {len(crit_findings)} critical Doctor finding(s)", - "detail": "Doctor surfaced critical readiness gaps in the repo.", - "cta": "Jump to AgentOps Doctor", + "title": f"Fix Doctor: {finding.get('title') or finding.get('id') or 'finding'}", + "detail": _render_recommendation_body(recommendation), + "cta": "Open Doctor finding", "anchor": "#section-agentops-doctor", } ) - official_eval = _official_eval_artifact_status(workspace) - has_eval_proof = ( - bool(eval_payload.get("has_runs")) - or bool(eval_payload.get("runs")) - or bool(official_eval.get("present")) + readiness_order = {"warn": 0, "muted": 1, "info": 2} + incomplete = sorted( + (check for check in readiness.get("checks", []) if check.get("status") != "ok"), + key=lambda item: readiness_order.get(str(item.get("status") or ""), 3), ) - if not has_eval_proof: + for check in incomplete: actions.append( { - "title": "Run your first evaluation", - "detail": ( - "No eval gate evidence yet. Run agentops eval run " - "locally, or run the generated official-eval workflow for a " - "compatible Foundry prompt agent." - ), - "cta": "agentops eval run", - } - ) - - evidence = _release_evidence_status(workspace) - if has_eval_proof and evidence.get("status") in {"missing", "unreadable"}: - actions.append( - { - "title": "Generate release evidence", - "detail": ( - "Package the latest eval gate, Doctor findings, CI/CD " - "status, and Foundry links into the release-review artifact." - ), - "cta": "agentops doctor --evidence-pack", + "title": f"Complete readiness: {check.get('title') or 'configuration'}", + "detail": check.get("detail", ""), + "cta": "Open readiness item", + "anchor": "#section-readiness", } ) @@ -2385,8 +2271,7 @@ def _build_next_actions( { "title": "All caught up", "detail": ( - "No outstanding readiness gaps detected in the repo. " - "Use the Foundry deep-links above to monitor runtime." + "All readiness items are complete and Doctor has no findings." ), "cta": None, } @@ -2610,6 +2495,142 @@ def _read_agentops_config(workspace: Path) -> Dict[str, Any]: return payload if isinstance(payload, dict) else {} +def _detect_foundry_agent_runtime( + workspace: Path, + agentops_config: Dict[str, Any], +) -> Optional[str]: + """Identify Foundry runtimes that provide native agent tracing.""" + path = workspace / "azure.yaml" + if path.is_file(): + try: + payload = load_yaml(path) + except Exception: + payload = None + if isinstance(payload, dict): + services = payload.get("services") + if isinstance(services, dict) and any( + isinstance(service, dict) + and service.get("host") == "azure.ai.agent" + and service.get("kind") == "hosted" + for service in services.values() + ): + return "hosted agent runtime" + + raw_agent = agentops_config.get("agent") + if isinstance(raw_agent, str) and raw_agent.strip(): + try: + from agentops.core.agentops_config import classify_agent + + target = classify_agent( + raw_agent, + protocol=agentops_config.get("protocol"), + ) + except (TypeError, ValueError): + target = None + if target is not None: + if target.kind == "foundry_hosted": + return "hosted agent runtime" + if target.kind == "foundry_prompt": + return "prompt agent runtime" + + for eval_path in (workspace / "src").glob("**/eval.y*ml"): + try: + eval_payload = load_yaml(eval_path) + except Exception: + continue + if not isinstance(eval_payload, dict): + continue + agent = eval_payload.get("agent") + kind = agent.get("kind") if isinstance(agent, dict) else None + if kind == "hosted": + return "hosted agent runtime" + if kind in {"prompt", "prompt-agent"}: + return "prompt agent runtime" + return None + + +def _detect_custom_tracing(workspace: Path) -> Optional[str]: + """Find optional repository code that emits custom OpenTelemetry spans.""" + candidates = list((workspace / "src").glob("**/*.py")) + candidates.extend(workspace.glob("*.py")) + for path in candidates: + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + has_otel = "opentelemetry" in text or "configure_azure_monitor" in text + emits_spans = any( + marker in text + for marker in ( + "start_as_current_span", + "start_span(", + "get_tracer(", + "configure_azure_monitor(", + ) + ) + if has_otel and emits_spans: + return path.relative_to(workspace).as_posix() + return None + + +def _detect_rubric_evaluator( + workspace: Path, + agentops_config: Dict[str, Any], +) -> Tuple[bool, str]: + """Resolve rubric evidence from AgentOps or azd AI Agent eval config.""" + rubrics = agentops_config.get("rubrics") + if isinstance(rubrics, list) and rubrics: + return True, "agentops.yaml" + + for path in (workspace / "src").glob("**/eval.y*ml"): + try: + payload = load_yaml(path) + except Exception: + continue + if not isinstance(payload, dict): + continue + evaluators = payload.get("evaluators") + if isinstance(evaluators, list) and any( + isinstance(evaluator, dict) + and bool(evaluator.get("local_uri") or evaluator.get("id")) + for evaluator in evaluators + ): + return True, path.relative_to(workspace).as_posix() + return False, "" + + +def _detect_alert_configuration( + workspace: Path, + agentops_config: Dict[str, Any], +) -> Tuple[bool, str]: + """Find explicit alert definitions without claiming cloud state.""" + observability = agentops_config.get("observability") + if isinstance(observability, dict) and observability.get("alerts"): + return True, "agentops.yaml" + + markers = ( + "microsoft.insights/metricalerts", + "microsoft.insights/scheduledqueryrules", + "azurerm_monitor_metric_alert", + "azurerm_monitor_scheduled_query_rules_alert", + ) + candidates: List[Path] = [] + for root in (workspace / "infra", workspace / "deploy"): + if root.is_dir(): + for pattern in ("**/*.bicep", "**/*.tf", "**/*.json", "**/*.y*ml"): + candidates.extend(root.glob(pattern)) + for pattern in ("*.bicep", "*.tf"): + candidates.extend(workspace.glob(pattern)) + for path in candidates: + try: + text = path.read_text(encoding="utf-8", errors="ignore").lower() + except OSError: + continue + if any(marker in text for marker in markers): + return True, path.relative_to(workspace).as_posix() + return False, "" + + def _read_trace_regression_manifest(workspace: Path) -> Dict[str, Any]: return _read_json_object(workspace / ".agentops" / "data" / "trace-regression-manifest.json") @@ -3468,10 +3489,8 @@ def _collapsible_section( def _render_status_cards_section( readiness: Dict[str, Any], watchdog: Dict[str, Any], - eval_payload: Dict[str, Any], ) -> str: - """Render the consolidated top status: three clickable cards that - answer "can I ship?" at a glance (Readiness, Doctor, Eval gate). + """Render the two go/no-go verdicts that answer "Can I ship?". Each card is an anchor to the matching detail section below; the template's hash-open script expands that (collapsed) section when the @@ -3503,9 +3522,9 @@ def _card( readiness_tone = "warn" readiness_card = _card( title="Readiness", - value=readiness_label, + value="GO" if readiness_tone == "ok" else "NO-GO", tone=readiness_tone, - sub="Observability checks green", + sub=readiness_label, anchor="#section-readiness", ) @@ -3521,9 +3540,9 @@ def _headline_value(key: str) -> int: return 0 if not watchdog.get("has_history"): - doctor_tone = "muted" - doctor_value = "No runs" - doctor_sub = "Run agentops doctor" + doctor_tone = "warn" + doctor_value = "NO-GO" + doctor_sub = "No Doctor run" else: findings_total = _headline_value("findings_total") critical = _headline_value("critical") @@ -3533,7 +3552,7 @@ def _headline_value(key: str) -> int: doctor_tone = "warn" else: doctor_tone = "ok" - doctor_value = f"{findings_total} finding(s)" + doctor_value = "GO" if findings_total == 0 else "NO-GO" doctor_sub = f"{critical} critical" doctor_card = _card( title="Doctor", @@ -3543,28 +3562,7 @@ def _headline_value(key: str) -> int: anchor="#section-agentops-doctor", ) - if not eval_payload.get("has_runs"): - eval_tone = "muted" - eval_value = "No runs" - eval_sub = "Run agentops eval run" - else: - passed = bool(eval_payload.get("latest_passed")) - eval_tone = "ok" if passed else "crit" - eval_value = "Pass" if passed else "Fail" - pass_rate = eval_payload.get("pass_rate") - if isinstance(pass_rate, (int, float)): - eval_sub = f"{int(pass_rate * 100)}% pass rate" - else: - eval_sub = "Latest run" - eval_card = _card( - title="Eval gate", - value=eval_value, - tone=eval_tone, - sub=eval_sub, - anchor="#section-eval-gates", - ) - - cards = readiness_card + doctor_card + eval_card + cards = readiness_card + doctor_card return ( '
' '
Can I ship? ' @@ -3636,7 +3634,7 @@ def _render_foundry_connection_section(connection: Dict[str, Any]) -> str: ) body = f'
{"".join(items_html)}
' return _collapsible_section( - "Foundry connection", body, section_id="section-foundry-connection" + "Connections", body, section_id="section-connections" ) @@ -3914,121 +3912,6 @@ def render_cockpit_html(payload: Dict[str, Any]) -> str: :func:`build_cockpit_payload`. Returns a complete HTML document. """ telemetry = payload["telemetry"] - # Show the telemetry card only when telemetry is OFF - it then acts as - # the "why is the production section empty" hint. When telemetry is - # active, the dedicated Production signal section communicates the - # connection state already. - show_telemetry_card = not telemetry.get("enabled", False) - telemetry_card = _render_telemetry_card(telemetry) if show_telemetry_card else "" - eval_runs_url = ( - telemetry.get("eval_runs_url") if isinstance(telemetry, dict) else None - ) - eval_link = "" - if eval_runs_url: - eval_link = ( - f' ' - f'View CI evals in App Insights →' - ) - - eval_body = "" - eval_subtitle = "Eval gate summary" - eval_caption = ( - '
' - 'AgentOps gate history from local artifacts and CI runs. For ' - 'Foundry cloud runs, use this as the quick pass/fail ' - 'triage view and open Foundry Evaluations for full analysis.' - '
' - ) - if payload["eval"]["has_runs"]: - cards_html = "".join(_render_card(c) for c in payload["eval"]["cards"]) - exec_tag = _render_exec_section_tag( - payload["eval"].get("latest_execution"), - ) - eval_body = f'{eval_caption}
{cards_html}{telemetry_card}
' - eval_subtitle = f"Eval gate summary{exec_tag}" - else: - official_eval = payload["eval"].get("official_eval") or {} - if official_eval.get("present"): - status = _html_escape(official_eval.get("status") or "metadata-only") - empty_text = ( - "No AgentOps-normalized eval runs yet under " - ".agentops/results/. Official Microsoft Foundry " - "AI Agent Evaluation evidence exists under " - f".agentops/official-eval/ with status {status}. " - "Run agentops doctor --evidence-pack to package it " - "for release review." - ) - else: - empty_text = ( - "No eval runs yet under .agentops/results/. " - "Run agentops eval run to populate this section." - ) - eval_body = ( - eval_caption + - '
' - f"{empty_text}" - "
" - + (f'
{telemetry_card}
' if telemetry_card else "") - ) - eval_subtitle = "Eval gate summary" - - deployments = payload.get("deployments") or {} - if deployments.get("has_data") and deployments.get("cards"): - deploy_cards = "".join(_render_card(c) for c in deployments["cards"]) - deployments_body = f'
{deploy_cards}
' - else: - hint = deployments.get("hint") or ( - "Install the GitHub CLI and run gh auth login " - "to surface workflow runs here." - ) - deployments_body = f'
{hint}
' - deployments_section = _collapsible_section( - "CI/CD Pipelines", deployments_body, section_id="section-cicd", - open_by_default=False, - ) - - metrics_subtitle = "Quality gate summary" - metrics_body = "" - if payload["metrics"]: - metrics_html = "".join(_render_card(c) for c in payload["metrics"]) - exec_tag = _render_exec_section_tag( - payload["eval"].get("latest_execution") if payload["eval"].get("has_runs") else None, - ) - metrics_caption = ( - '
' - 'Quality gate trends computed from AgentOps result artifacts. ' - 'Keep this as a compact threshold/regression summary; detailed ' - 'cloud-evaluation drilldown belongs in Foundry Evaluations.' - '
' - ) - metrics_body = f'{metrics_caption}
{metrics_html}
' - metrics_subtitle = f"Quality gate summary{exec_tag}" - - # Merge the eval gate + quality gate into a single "Eval gates" section - # with two subgroups. They share a source (AgentOps result artifacts) - # and CTA (View CI evals in App Insights), so folding them removes a - # redundant top-level section and keeps the gate story in one place. - eval_gate_subgroup = ( - '
' - f'
{eval_subtitle}
' - f'{eval_body}' - '
' - ) - quality_gate_subgroup = "" - if metrics_body: - quality_gate_subgroup = ( - '
' - f'
{metrics_subtitle}
' - f'{metrics_body}' - '
' - ) - eval_gates_section = _collapsible_section( - f"Eval gates{eval_link}", - eval_gate_subgroup + quality_gate_subgroup, - section_id="section-eval-gates", - open_by_default=False, - ) watchdog = payload["watchdog"] watchdog_title = "AgentOps Doctor" @@ -4063,114 +3946,9 @@ def render_cockpit_html(payload: Dict[str, Any]) -> str: open_by_default=False, ) - production = payload.get("production") or {} - production_section = "" - # The App Insights KQL portal URL is already surfaced by the launchpad - # "App Insights" tile and the Doctor / Eval gate "View in App Insights" - # links, so we no longer repeat it here. Production signal keeps a - # single primary CTA: the full Foundry Monitor view. - _portal_url_unused = telemetry.get("portal_url") if isinstance(telemetry, dict) else None - - # Cockpit surfaces a 2-card teaser (error rate + P95); this link is - # the primary call-to-action for the full Foundry Monitor view. - foundry_monitor_url = None - open_panel = payload.get("open_in_foundry") or {} - for target in open_panel.get("targets", []): - if target.get("key") == "monitor": - foundry_monitor_url = target.get("url") - break - foundry_monitor_link = "" - if foundry_monitor_url: - foundry_monitor_link = ( - f' ' - 'Full view in Foundry Monitor →' - ) - - prod_title = ( - 'Production signal' - ' live · App Insights' - f'{foundry_monitor_link}' - ) - if production.get("has_data") and production.get("cards"): - # Server-side render (rare - happens when /api/production/html is - # invoked directly without a deferred placeholder). - prod_html = "".join(_render_card(c, hero=True) for c in production["cards"]) - prod_caption = ( - '
' - 'Fast health snapshot from App Insights. Use ' - 'Foundry Monitor for the full production view ' - '(invocations, tokens, per-model breakdown). Raw App Insights ' - 'KQL is available from the launchpad "App Insights" tile.' - '
' - ) - prod_body = f'{prod_caption}
{prod_html}
' - production_section = _collapsible_section( - prod_title, prod_body, open_by_default=False, - ) - elif production.get("deferred"): - # Telemetry is wired up; the cards will arrive async from - # /api/production/html so the page can render immediately. - # Render 2 skeleton cards matching the teaser layout - # (Error rate / P95 latency). Invocations and tokens - # intentionally live in Foundry Monitor only. - skeleton_labels = ("Error rate", "P95 latency") - skeleton_cards = "".join( - ( - '
' - f'
{label}
' - '
' - '
' - '
' - '
' - ) - for label in skeleton_labels - ) - prod_caption = ( - '
' - 'Fast health snapshot from App Insights. Use ' - 'Foundry Monitor for the full production view ' - '(invocations, tokens, per-model breakdown). Raw App Insights ' - 'KQL is available from the launchpad "App Insights" tile.' - '
' - ) - prod_body = ( - f'{prod_caption}' - f'
{skeleton_cards}
' - ) - production_section = _collapsible_section( - prod_title, prod_body, open_by_default=False, - ) - - counts = payload["summary_counts"] workspace_display = _shorten_workspace(payload["workspace"]) - range_info = payload.get("time_range") or {} - range_bar = _render_range_bar(range_info) - - foundry_uri = _foundry_logo_data_uri() - foundry_url = payload.get("foundry_project_url") or "https://ai.azure.com" - if foundry_uri: - powered_by_html = ( - f'' - f'Foundry' - 'Your Foundry project ↗' - '' - ) - else: - powered_by_html = "" - - # Banner removed; the primer click is now folded into the project - # action button above, which carries the ?tid= hint. - setup_banner_html = "" - - foundry_connection_section = _render_foundry_connection_section( - payload.get("foundry_connection") or {"items": []} - ) - open_in_foundry_section = _render_open_in_foundry_section( - payload.get("open_in_foundry") or {"targets": []} + connections_section = _render_foundry_connection_section( + payload.get("connections") or {"items": []} ) readiness_section = _render_readiness_section( payload.get("readiness") or {"checks": [], "label": "0/0 ready"} @@ -4181,98 +3959,20 @@ def render_cockpit_html(payload: Dict[str, Any]) -> str: status_cards_section = _render_status_cards_section( payload.get("readiness") or {"checks": [], "label": "0/0 ready"}, payload.get("watchdog") or {}, - payload.get("eval") or {}, ) return _COCKPIT_TEMPLATE.format( - foundry_connection_section=foundry_connection_section, - open_in_foundry_section=open_in_foundry_section, status_cards_section=status_cards_section, + connections_section=connections_section, readiness_section=readiness_section, - next_actions_section=next_actions_section, - eval_gates_section=eval_gates_section, - deployments_section=deployments_section, - production_section=production_section, watchdog_section=watchdog_section, - eval_runs=counts["eval_runs"], - analyses=counts["analyses"], + next_actions_section=next_actions_section, workspace_display=workspace_display, workspace=payload["workspace"], icon_uri=_icon_data_uri(), - powered_by=powered_by_html, - setup_banner=setup_banner_html, - range_bar=range_bar, - range_label=_html_escape(range_info.get("label", "")), - ) - - -def _render_range_bar(range_info: Dict[str, Any]) -> str: - """Render the 1D / 7D / 30D / Custom selector.""" - active_key = range_info.get("key", "7d") - pills: List[str] = [] - labels = {"1d": "1D", "7d": "7D", "30d": "30D"} - for key in preset_keys(): - cls = "range-pill active" if key == active_key else "range-pill" - pills.append(f'{labels[key]}') - custom_cls = "range-pill active" if active_key == "custom" else "range-pill" - pills.append( - f'Custom' - ) - - today = _today_iso() - week_ago = _days_ago_iso(7) - custom_from = range_info.get("start", "")[:10] if active_key == "custom" else week_ago - custom_to = range_info.get("end", "")[:10] if active_key == "custom" else today - form_class = "range-custom-form open" if active_key == "custom" else "range-custom-form" - - custom_form = ( - f'
' - f'' - f'' - f'' - f'' - f'
' - ) - - refresh_control = ( - '
' - '' - 'Refresh' - '' - '
' - ) - - return ( - '
' - + '
' + "".join(pills) + '
' - + custom_form - + refresh_control - + '
' ) -def _today_iso() -> str: - from datetime import datetime, timezone - return datetime.now(timezone.utc).strftime("%Y-%m-%d") - - -def _days_ago_iso(days: int) -> str: - from datetime import datetime, timedelta, timezone - return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d") - - def _shorten_workspace(path: str) -> str: """Show only the current folder name for compact heading display. @@ -4736,91 +4436,6 @@ def _shorten_workspace(path: str) -> str: 0%, 100% {{ opacity: 1; }} 50% {{ opacity: 0.6; }} }} - .range-bar {{ - display: flex; align-items: center; flex-wrap: wrap; gap: 12px; - margin-bottom: 8px; - }} - .range-pills {{ display: flex; gap: 4px; }} - .range-pill {{ - padding: 6px 14px; border-radius: 999px; - color: var(--text-dim); text-decoration: none; - font-size: 12px; font-weight: 600; letter-spacing: 0.02em; - background: rgba(255, 255, 255, 0.03); - border: 1px solid var(--border); - transition: background 0.15s ease, color 0.15s ease; - }} - .range-pill:hover {{ - background: rgba(255, 255, 255, 0.06); color: var(--text); - }} - .range-pill.active {{ - background: rgba(56, 189, 248, 0.14); color: var(--info); - border-color: rgba(56, 189, 248, 0.35); - }} - .range-custom-form {{ - display: none; gap: 10px; align-items: center; - background: var(--card); padding: 10px 14px; - border-radius: 999px; border: 1px solid var(--border); - font-size: 12px; color: var(--text-dim); - }} - .range-custom-form.open {{ display: flex; }} - .range-custom-form input[type="date"] {{ - background: rgba(255, 255, 255, 0.04); - border: 1px solid var(--border); color: var(--text); - padding: 4px 8px; border-radius: 8px; font-size: 12px; - color-scheme: dark; - }} - .range-custom-form button {{ - background: var(--info); color: #08090b; border: 0; - padding: 5px 12px; border-radius: 999px; - font-size: 12px; font-weight: 700; cursor: pointer; - }} - .range-current {{ - color: var(--text-faint); font-size: 11px; font-weight: 500; - margin-left: auto; - font-family: "SF Mono", "Cascadia Code", Consolas, monospace; - }} - .refresh-control {{ - margin-left: auto; - display: inline-flex; align-items: center; gap: 8px; - padding: 5px 10px 5px 12px; border-radius: 999px; - background: rgba(255, 255, 255, 0.03); - border: 1px solid var(--border); - color: var(--text-dim); - font-size: 12px; font-weight: 600; letter-spacing: 0.02em; - transition: background 0.15s ease, color 0.15s ease, - border-color 0.15s ease; - }} - .refresh-control:hover {{ - background: rgba(255, 255, 255, 0.06); color: var(--text); - border-color: rgba(56, 189, 248, 0.35); - }} - .refresh-control .refresh-icon {{ - color: var(--info); opacity: 0.9; - }} - .refresh-control.spinning .refresh-icon {{ - animation: refresh-spin 0.8s linear; - }} - .refresh-label {{ - color: var(--text-faint); text-transform: uppercase; - font-size: 10px; letter-spacing: 0.06em; - }} - #refreshSelect {{ - appearance: none; -webkit-appearance: none; - background: transparent; color: var(--text); - border: 0; outline: none; - font: inherit; font-weight: 600; - padding: 0 18px 0 2px; cursor: pointer; - background-image: url("data:image/svg+xml;utf8,"); - background-repeat: no-repeat; - background-position: right 2px center; - }} - #refreshSelect option {{ - background: var(--card); color: var(--text); - }} - @keyframes refresh-spin {{ - from {{ transform: rotate(0deg); }} - to {{ transform: rotate(360deg); }} - }} .grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 14px; @@ -5207,84 +4822,17 @@ def _shorten_workspace(path: str) -> str:
{workspace_display}
-
-
-
{eval_runs} eval(s)
-
{analyses} analysis run(s)
-
- {powered_by} -
-{setup_banner} - -{range_bar} -
window: {range_label}
- -{foundry_connection_section} {status_cards_section} -{next_actions_section} +{connections_section} {readiness_section} {watchdog_section} -{eval_gates_section} -{production_section} -{deployments_section} -{open_in_foundry_section} +{next_actions_section} -
Auto-refresh: every 5 min · agentops cockpit
+
agentops cockpit
@@ -5425,21 +4913,12 @@ def create_app(workspace: Path): @app.get("/", response_class=HTMLResponse) def _index( - range_: Optional[str] = Query(None, alias="range"), - from_: Optional[str] = Query(None, alias="from"), - to: Optional[str] = Query(None), partial: Optional[str] = Query(None, alias="_partial"), ) -> HTMLResponse: - # The full render does file IO, history aggregation, and a - # `gh run list` subprocess for CI/CD, which together can take - # several seconds. To avoid a "black page" while the browser - # waits, ``/`` returns a tiny branded shell with an animated - # loader, and the shell fetches ``/?_partial=1`` to hydrate - # the real cockpit once the heavy work completes. + # Return a tiny shell immediately, then hydrate the local cockpit. if not partial: return HTMLResponse(_render_loading_shell()) - time_range = parse_time_range(range_, from_, to) - payload = build_cockpit_payload(workspace, time_range=time_range) + payload = build_cockpit_payload(workspace) return HTMLResponse(render_cockpit_html(payload)) @app.get("/favicon.ico") @@ -5468,25 +4947,6 @@ def _api_run_report(run_id: str) -> HTMLResponse: def _api_telemetry() -> JSONResponse: return JSONResponse(_telemetry_status()) - @app.get("/api/production") - def _api_production( - range_: Optional[str] = Query(None, alias="range"), - from_: Optional[str] = Query(None, alias="from"), - to: Optional[str] = Query(None), - ) -> JSONResponse: - time_range = parse_time_range(range_, from_, to) - return JSONResponse(_build_production_section(_telemetry_status(), time_range=time_range)) - - @app.get("/api/production/html", response_class=HTMLResponse) - def _api_production_html( - range_: Optional[str] = Query(None, alias="range"), - from_: Optional[str] = Query(None, alias="from"), - to: Optional[str] = Query(None), - ) -> HTMLResponse: - time_range = parse_time_range(range_, from_, to) - production = _build_production_section(_telemetry_status(), time_range=time_range) - return HTMLResponse(render_production_grid_html(production)) - @app.get("/healthz") def _healthz() -> Dict[str, str]: return {"status": "ok"} diff --git a/src/agentops/agent/sources/azure_monitor.py b/src/agentops/agent/sources/azure_monitor.py index 42185e3..202e576 100644 --- a/src/agentops/agent/sources/azure_monitor.py +++ b/src/agentops/agent/sources/azure_monitor.py @@ -105,12 +105,29 @@ def collect_azure_monitor( ) -> AzureMonitorPayload: """Run KQL queries against Application Insights for the lookback window.""" diagnostics: Dict[str, Any] = {"enabled": config.enabled} + app_insights_resource_id = config.app_insights_resource_id if not config.enabled: diagnostics["status"] = "disabled" return AzureMonitorPayload(diagnostics=diagnostics) - if not config.app_insights_resource_id and not config.log_analytics_workspace_id: + if not app_insights_resource_id and not config.log_analytics_workspace_id: + resource_reason: Optional[str] = None + try: + from agentops.utils.foundry_discovery import ( + resolve_appinsights_resource_id_from_env_with_reason, + ) + + app_insights_resource_id, resource_reason = ( + resolve_appinsights_resource_id_from_env_with_reason() + ) + except Exception as exc: # noqa: BLE001 + resource_reason = f"Foundry App Insights metadata discovery failed: {exc}" + + if app_insights_resource_id: + diagnostics["target_source"] = "foundry_project_connection" + + if not app_insights_resource_id and not config.log_analytics_workspace_id: application_id, source, reason = _resolve_application_id() if application_id: diagnostics["target"] = application_id @@ -127,8 +144,9 @@ def collect_azure_monitor( "is configured, and no App Insights ApplicationId could be " "discovered from the connection string or Foundry project" ) - if reason: - diagnostics["discovery_reason"] = reason + discovery_reason = reason or resource_reason + if discovery_reason: + diagnostics["discovery_reason"] = discovery_reason return AzureMonitorPayload(diagnostics=diagnostics) try: @@ -146,7 +164,7 @@ def collect_azure_monitor( from ._credentials import format_source_error, get_shared_credential, log_source_error # noqa: F401 workspace_or_resource = ( - config.log_analytics_workspace_id or config.app_insights_resource_id + config.log_analytics_workspace_id or app_insights_resource_id ) diagnostics["target"] = workspace_or_resource @@ -175,7 +193,7 @@ def collect_azure_monitor( ) return AzureMonitorPayload(diagnostics=diagnostics) response = query_resource( - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=kql, timespan=None, ) @@ -224,7 +242,7 @@ def collect_azure_monitor( ) else: safety_response = client.query_resource( # type: ignore[union-attr] - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=safety_kql, timespan=None, ) @@ -265,7 +283,7 @@ def collect_azure_monitor( ) else: tok_response = client.query_resource( # type: ignore[union-attr] - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=token_kql, timespan=None, ) @@ -300,7 +318,7 @@ def collect_azure_monitor( ) else: rl_response = client.query_resource( # type: ignore[union-attr] - resource_id=config.app_insights_resource_id, + resource_id=app_insights_resource_id, query=rl_kql, timespan=None, ) diff --git a/src/agentops/cli/app.py b/src/agentops/cli/app.py index 539d72e..5eacc95 100644 --- a/src/agentops/cli/app.py +++ b/src/agentops/cli/app.py @@ -79,16 +79,6 @@ "for the manual." ) ) -telemetry_app = typer.Typer( - help="Import Azure Monitor telemetry into AgentOps datasets." -) -dashboard_app = typer.Typer( - help=( - "Deploy, open, and export the Foundry operations Azure Monitor " - "workbook (capacity, traffic and tokens, latency, errors, agent behavior)." - ) -) -telemetry_app.add_typer(dashboard_app, name="dashboard") app.add_typer(eval_app, name="eval") app.add_typer(report_app, name="report") app.add_typer(workflow_app, name="workflow") @@ -100,7 +90,6 @@ app.add_typer(init_app, name="init") app.add_typer(assert_app, name="assert") app.add_typer(redteam_app, name="redteam") -app.add_typer(telemetry_app, name="telemetry") log = get_logger(__name__) DEFAULT_REPORT_INPUT = Path(".agentops/results/latest/results.json") @@ -829,85 +818,6 @@ class ExplainPage: outputs=("`.github/skills/agentops-*` for Copilot", "`.claude/commands/agentops-*` for Claude Code"), examples=("agentops skills install", "agentops skills install --platform copilot", "agentops skills install --from github:org/repo@v1"), ), - ("telemetry", "dashboard"): ExplainPage( - title="Foundry operations dashboard", - command="agentops telemetry dashboard", - synopsis=("agentops telemetry dashboard COMMAND [ARGS]...", "agentops telemetry dashboard explain"), - summary=( - "Deploys, opens, and exports the Foundry operations Azure Monitor " - "workbook: capacity (PTU), traffic and tokens, latency percentiles, " - "errors and throttling, and read-only Foundry trace-evaluation " - "behavior for an Azure OpenAI resource.", - "The workbook is scoped per Azure OpenAI resource and per Log " - "Analytics workspace and reads from AzureMetrics, AzureDiagnostics, " - "and Foundry-owned Application Insights events and spans.", - ), - children=("deploy", "open", "export"), - ), - ("telemetry", "dashboard", "deploy"): ExplainPage( - title="Deploy the Foundry operations workbook", - command="agentops telemetry dashboard deploy", - synopsis=( - "agentops telemetry dashboard deploy [--dry-run] [--subscription ID] " - "[--resource-group RG] [--workspace-id ID] [--name NAME] [--dir PATH]", - "agentops telemetry dashboard deploy explain", - ), - summary=( - "Deploys the workbook as a Microsoft.Insights/workbooks ARM resource " - "into the discovered (or supplied) resource group.", - "This is the first AgentOps CLI command that creates an Azure " - "resource; it deploys a single workbook and nothing else.", - ), - how_it_works=( - "Discovers subscription, resource group, Log Analytics workspace, " - "and the Azure OpenAI resource from agentops.yaml and the azd env.", - "Runs an RBAC preflight: Workbook Contributor on the resource group " - "and Log Analytics Reader on the workspace. Missing roles fail with " - "the exact role and scope to request.", - "Warns (non-fatally) and prints the exact " - "`az monitor diagnostic-settings create` command when the Azure " - "OpenAI resource is missing the RequestResponse or " - "AzureOpenAIRequestUsage categories.", - "Deploys via `az deployment group create` and prints the portal URL.", - ), - outputs=("A deployed workbook and its Azure portal URL", "The ARM template when --dry-run is used"), - examples=( - "agentops telemetry dashboard deploy --dry-run", - "agentops telemetry dashboard deploy --resource-group my-rg", - ), - ), - ("telemetry", "dashboard", "open"): ExplainPage( - title="Open the Foundry operations workbook", - command="agentops telemetry dashboard open", - synopsis=( - "agentops telemetry dashboard open [--print-url] [--subscription ID] " - "[--resource-group RG] [--name NAME] [--dir PATH]", - "agentops telemetry dashboard open explain", - ), - summary=( - "Builds the Azure portal URL for the workbook and opens it in the " - "default browser.", - "In a non-interactive shell, or with --print-url, it prints the URL " - "instead of opening a browser.", - ), - examples=( - "agentops telemetry dashboard open", - "agentops telemetry dashboard open --print-url", - ), - ), - ("telemetry", "dashboard", "export"): ExplainPage( - title="Export the workbook JSON", - command="agentops telemetry dashboard export", - synopsis=( - "agentops telemetry dashboard export [--out PATH]", - "agentops telemetry dashboard export explain", - ), - summary=( - "Copies the packaged workbook JSON to a local path so you can import " - "it manually or customize it before deploying.", - ), - examples=("agentops telemetry dashboard export --out foundry-ops.workbook.json",), - ), ("mcp",): ExplainPage( title="MCP commands", command="agentops mcp", @@ -1589,7 +1499,6 @@ def _cmd( prompt_app.command("explain")(_make_group_explain(("prompt",))) mcp_app.command("explain")(_make_group_explain(("mcp",))) agent_app.command("explain")(_make_group_explain(("agent",))) -dashboard_app.command("explain")(_make_group_explain(("telemetry", "dashboard"))) # --------------------------------------------------------------------------- @@ -2485,306 +2394,12 @@ def cmd_eval_promote_traces( ) -@telemetry_app.command("validate") -def cmd_telemetry_validate( - name: Annotated[str, typer.Argument(help="Name under telemetry_imports.")], - config: Annotated[ - Optional[Path], - typer.Option("--config", "-c", help="Path to agentops.yaml."), - ] = None, -) -> None: - """Validate a named telemetry import without querying Azure.""" - - from agentops.core.config_loader import load_agentops_config - from agentops.services.telemetry_import import ( - TelemetryImportError, - find_telemetry_import, - validate_telemetry_import, - ) - - try: - cfg = load_agentops_config(_resolve_eval_config_path(config)) - item = find_telemetry_import(cfg, name) - warnings = validate_telemetry_import(item) - except (TelemetryImportError, ValueError) as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(1) from exc - typer.echo(_cli_ok(f"telemetry import {name!r} is valid")) - for warning in warnings: - typer.echo(_cli_warn(f"warning: {warning}")) - - -@telemetry_app.command("preview") -def cmd_telemetry_preview( - name: Annotated[str, typer.Argument(help="Name under telemetry_imports.")], - rows: Annotated[int, typer.Option("--rows", min=1, help="Maximum rows to preview.")] = 10, - config: Annotated[ - Optional[Path], - typer.Option("--config", "-c", help="Path to agentops.yaml."), - ] = None, -) -> None: - """Query Azure Monitor and print a small dataset preview.""" - - from agentops.core.config_loader import load_agentops_config - from agentops.services.telemetry_import import ( - TelemetryImportError, - find_telemetry_import, - preview_telemetry_import, - render_telemetry_import_preview, - ) - - try: - cfg = load_agentops_config(_resolve_eval_config_path(config)) - item = find_telemetry_import(cfg, name) - preview = preview_telemetry_import(item, rows=rows, apply=False) - except (TelemetryImportError, ValueError) as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(1) from exc - typer.echo(render_telemetry_import_preview(preview)) - - -@telemetry_app.command("import") -def cmd_telemetry_import( - name: Annotated[str, typer.Argument(help="Name under telemetry_imports.")], - apply: Annotated[ - bool, - typer.Option("--apply", help="Write JSONL rows and manifest."), - ] = False, - rows: Annotated[ - Optional[int], - typer.Option("--rows", min=1, help="Optional maximum rows to import."), - ] = None, - config: Annotated[ - Optional[Path], - typer.Option("--config", "-c", help="Path to agentops.yaml."), - ] = None, -) -> None: - """Import telemetry into the configured JSONL output path.""" - - from agentops.core.config_loader import load_agentops_config - from agentops.services.telemetry_import import ( - TelemetryImportError, - find_telemetry_import, - preview_telemetry_import, - render_telemetry_import_preview, - ) - - if not apply: - typer.echo( - _cli_warn( - "Dry run only. Re-run with --apply to write the JSONL dataset and manifest." - ) - ) - try: - cfg = load_agentops_config(_resolve_eval_config_path(config)) - item = find_telemetry_import(cfg, name) - preview = preview_telemetry_import(item, rows=rows, apply=apply) - except (TelemetryImportError, ValueError) as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(1) from exc - typer.echo(render_telemetry_import_preview(preview)) - if apply: - typer.echo(_cli_updated(preview.output_path)) - typer.echo(_cli_updated(preview.manifest_path)) - - def _resolve_eval_config_path(config: Path | None) -> Path: if config is not None: return config return Path("agentops.yaml") -# --------------------------------------------------------------------------- -# agentops telemetry dashboard {deploy, open, export} -# --------------------------------------------------------------------------- -@dashboard_app.command("deploy") -def cmd_dashboard_deploy( - dry_run: Annotated[ - bool, - typer.Option("--dry-run", help="Emit the ARM template and make no changes."), - ] = False, - subscription: Annotated[ - Optional[str], - typer.Option("--subscription", help="Azure subscription id override."), - ] = None, - resource_group: Annotated[ - Optional[str], - typer.Option("--resource-group", help="Resource group for the workbook."), - ] = None, - workspace_id: Annotated[ - Optional[str], - typer.Option("--workspace-id", help="Log Analytics workspace resource id."), - ] = None, - name: Annotated[ - Optional[str], - typer.Option("--name", help="Workbook display name."), - ] = None, - workspace: Annotated[ - Path, - typer.Option("--dir", help="AgentOps workspace root for discovery."), - ] = Path("."), - explain: Annotated[str | None, typer.Argument(hidden=True)] = None, -) -> None: - """Deploy the Foundry operations workbook to Azure Monitor.""" - - if _maybe_explain_leaf(("telemetry", "dashboard", "deploy"), explain): - return - - import json - - from agentops.services import dashboard as dash - - target = dash.discover_target( - workspace.resolve(), - subscription_id=subscription, - resource_group=resource_group, - workspace_id=workspace_id, - name=name, - ) - - if dry_run: - template = dash.build_arm_template(target=target) - typer.echo(json.dumps(template, indent=2)) - typer.echo( - _cli_warn( - "Dry run only. No Azure changes were made. Re-run without " - "--dry-run to deploy." - ), - err=True, - ) - return - - # RBAC preflight — fail gracefully with the exact role and scope needed. - rbac = dash.check_rbac( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - workspace_id=target.workspace_id, - ) - for message in rbac.messages: - if rbac.level == "ok": - typer.echo(_cli_ok(message)) - elif rbac.level == "warn": - typer.echo(_cli_warn(message), err=True) - else: - typer.echo(_cli_error(message), err=True) - if not rbac.ok: - raise typer.Exit(code=1) - - # Diagnostic-settings advisory (non-fatal): print the exact fix command. - enabled = list(target.discovery.get("enabled_log_categories", []) or []) - missing = dash.missing_diagnostic_categories(enabled) if enabled else list( - dash.REQUIRED_DIAGNOSTIC_CATEGORIES - ) - if missing: - typer.echo( - _cli_warn( - "The Azure OpenAI resource may not emit the categories the " - f"workbook needs ({', '.join(missing)}). If the tiles are " - "empty, enable them with:" - ), - err=True, - ) - typer.echo( - _cli_command( - dash.build_diagnostic_settings_command( - aoai_resource_id=target.aoai_resource_id, - workspace_id=target.workspace_id, - ) - ), - err=True, - ) - - try: - url = dash.deploy_workbook(target=target) - except dash.DashboardError as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(code=1) from exc - - typer.echo(_cli_ok("Workbook deployed.")) - typer.echo(f"{_cli_label('Portal')}: {_cli_path(url)}") - - -@dashboard_app.command("open") -def cmd_dashboard_open( - print_url: Annotated[ - bool, - typer.Option("--print-url", help="Print the URL instead of opening a browser."), - ] = False, - subscription: Annotated[ - Optional[str], - typer.Option("--subscription", help="Azure subscription id override."), - ] = None, - resource_group: Annotated[ - Optional[str], - typer.Option("--resource-group", help="Resource group for the workbook."), - ] = None, - name: Annotated[ - Optional[str], - typer.Option("--name", help="Workbook display name."), - ] = None, - workspace: Annotated[ - Path, - typer.Option("--dir", help="AgentOps workspace root for discovery."), - ] = Path("."), - explain: Annotated[str | None, typer.Argument(hidden=True)] = None, -) -> None: - """Open the Foundry operations workbook in the Azure portal.""" - - if _maybe_explain_leaf(("telemetry", "dashboard", "open"), explain): - return - - from agentops.services import dashboard as dash - - target = dash.discover_target( - workspace.resolve(), - subscription_id=subscription, - resource_group=resource_group, - name=name, - ) - url = dash.build_workbook_portal_url( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - name=target.name, - tenant_id=target.tenant_id, - ) - - if print_url or not _stream_is_interactive(sys.stdout): - typer.echo(url) - return - typer.echo(f"{_cli_heading('Foundry operations dashboard')} → {_cli_path(url)}") - try: - webbrowser.open(url) - except Exception: # noqa: BLE001 - best effort - typer.echo(url) - - -@dashboard_app.command("export") -def cmd_dashboard_export( - out: Annotated[ - Path, - typer.Option("--out", help="Destination path for the workbook JSON."), - ] = Path("foundry-ops.workbook.json"), - explain: Annotated[str | None, typer.Argument(hidden=True)] = None, -) -> None: - """Export the packaged workbook JSON to a local path.""" - - if _maybe_explain_leaf(("telemetry", "dashboard", "export"), explain): - return - - from agentops.services import dashboard as dash - - try: - content = dash.load_workbook_template() - except dash.DashboardError as exc: - typer.echo(_cli_error(str(exc)), err=True) - raise typer.Exit(code=1) from exc - - destination = out.resolve() - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - typer.echo(_cli_updated(destination)) - - def _append_assert_step_summary(result, *, scored_cases, pass_rate) -> None: """Append an ASSERT gate summary to the GitHub Actions step summary.""" from agentops.core.step_summary import append_step_summary, is_active diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py index 2728a37..1276b91 100644 --- a/src/agentops/core/agentops_config.py +++ b/src/agentops/core/agentops_config.py @@ -85,11 +85,6 @@ #: Where the local evaluator runtime gets the response text for each row. ResponseSource = Literal["agent", "dataset"] -#: Production telemetry import providers and destinations. -TelemetrySourceProvider = Literal["azure-monitor"] -TelemetryTarget = Literal["application-insights", "log-analytics"] -TelemetryLabelMode = Literal["self-similarity", "pending"] - #: Internal-only literal kept for the publisher dispatch table. Derived from #: ``execution`` + ``publish`` via :meth:`AgentOpsConfig.publish_target`. PublishTarget = Literal["foundry", "foundry_cloud"] @@ -391,116 +386,6 @@ def _url_non_empty(cls, value: Optional[str]) -> Optional[str]: return value -# --------------------------------------------------------------------------- -# Telemetry import configuration -# --------------------------------------------------------------------------- - - -class TelemetryTimeRangeConfig(BaseModel): - """Time window for a telemetry import query. - - Users can either provide explicit ISO-ish ``from``/``to`` timestamps or a - relative ``lookback_days`` window. The service owns final KQL rendering so - users never pass arbitrary query text. - """ - - from_: Optional[str] = Field(None, alias="from") - to: Optional[str] = None - lookback_days: Optional[int] = Field(None, ge=1, le=90) - - model_config = ConfigDict(extra="forbid", populate_by_name=True) - - @model_validator(mode="after") - def _validate_window(self) -> "TelemetryTimeRangeConfig": - explicit = self.from_ is not None or self.to is not None - if explicit and not (self.from_ and self.to): - raise ValueError("telemetry_imports.time_range requires both from and to") - if explicit and self.lookback_days is not None: - raise ValueError("telemetry_imports.time_range cannot mix from/to with lookback_days") - if not explicit and self.lookback_days is None: - self.lookback_days = 7 - return self - - -class TelemetryPrivacyConfig(BaseModel): - """Privacy controls applied before JSONL rows are written.""" - - redact_fields: List[str] = Field( - default_factory=lambda: ["authorization", "api_key", "token", "password", "secret"], - description="Case-insensitive field-name fragments to redact.", - ) - max_field_length: int = Field(4000, ge=100, le=20000) - include_raw: bool = False - - model_config = ConfigDict(extra="forbid") - - -class TelemetryOutputConfig(BaseModel): - """Output paths and labeling mode for generated dataset rows.""" - - path: Path = Field(Path(".agentops") / "data" / "telemetry-import.jsonl") - manifest_path: Optional[Path] = None - label_mode: TelemetryLabelMode = "self-similarity" - - model_config = ConfigDict(extra="forbid") - - -class TelemetryImportConfig(BaseModel): - """Named telemetry import declaration. - - The MVP intentionally keeps this declarative: users choose a supported - source/destination pair, field mappings, filters, privacy settings, and an - output file. The service generates the KQL. - """ - - name: str - source: TelemetrySourceProvider = "azure-monitor" - target: TelemetryTarget - resource_id: Optional[str] = None - workspace_id: Optional[str] = None - application_id: Optional[str] = None - connection_string: Optional[str] = None - time_range: TelemetryTimeRangeConfig = Field(default_factory=TelemetryTimeRangeConfig) - filters: Dict[str, str | List[str]] = Field(default_factory=dict) - fields: Dict[str, str] = Field(default_factory=dict) - privacy: TelemetryPrivacyConfig = Field(default_factory=TelemetryPrivacyConfig) - output: TelemetryOutputConfig = Field(default_factory=TelemetryOutputConfig) - max_rows: int = Field(100, ge=1, le=5000) - - model_config = ConfigDict(extra="forbid") - - @field_validator("name") - @classmethod - def _name_non_empty(cls, value: str) -> str: - value = value.strip() - if not value: - raise ValueError("telemetry_imports.name must be non-empty") - return value - - @field_validator("resource_id", "workspace_id", "application_id", "connection_string") - @classmethod - def _optional_text_non_empty(cls, value: Optional[str]) -> Optional[str]: - if value is None: - return value - value = value.strip() - if not value: - raise ValueError("telemetry_imports resource identifiers must be non-empty") - return value - - @model_validator(mode="after") - def _validate_target_ids(self) -> "TelemetryImportConfig": - if self.target == "log-analytics" and not self.workspace_id: - raise ValueError("telemetry_imports targeting log-analytics require workspace_id") - if self.target == "application-insights" and not ( - self.resource_id or self.application_id or self.connection_string - ): - raise ValueError( - "telemetry_imports targeting application-insights require resource_id, " - "application_id, or connection_string" - ) - return self - - class PromptAgentBootstrap(BaseModel): """Bootstrap defaults for prompt-agent CI/CD when the target Foundry project does not yet contain the seed agent referenced by ``agent``. @@ -1048,10 +933,6 @@ class AgentOpsConfig(BaseModel): ) evaluators: Optional[List[EvaluatorOverride]] = None - telemetry_imports: List[TelemetryImportConfig] = Field( - default_factory=list, - description="Named Azure Monitor imports that generate AgentOps JSONL datasets.", - ) rubrics: List[RubricConfig] = Field( default_factory=list, description="Optional context-specific rubric evaluator definitions.", diff --git a/src/agentops/services/agent_identity.py b/src/agentops/services/agent_identity.py index 3c3f29b..71f7f1a 100644 --- a/src/agentops/services/agent_identity.py +++ b/src/agentops/services/agent_identity.py @@ -518,10 +518,15 @@ def _name_from_target(workspace: Path) -> Optional[str]: if not isinstance(agent, str): return None raw = agent.strip() - if not raw or "://" in raw: + if not raw: return None - name = raw.split(":", 1)[0].strip() - return name or None + try: + from agentops.core.agentops_config import classify_agent + + target = classify_agent(raw) + except ValueError: + return None + return target.name def resolve_registration_inputs( diff --git a/src/agentops/services/dashboard.py b/src/agentops/services/dashboard.py deleted file mode 100644 index e694b8e..0000000 --- a/src/agentops/services/dashboard.py +++ /dev/null @@ -1,506 +0,0 @@ -"""Thin service layer for the Foundry operations Azure Monitor workbook. - -This module is the single place that knows how to: - -* load the packaged ``foundry-ops.workbook.json`` gallery template, -* build the Azure portal deep link for the deployed workbook, -* run the RBAC / diagnostic-settings preflight for ``agentops telemetry - dashboard deploy``, and -* wrap the workbook content into a ``Microsoft.Insights/workbooks`` ARM - template and (optionally) deploy it via the Azure CLI. - -Azure SDK / CLI access is kept lazy and fail-open so importing this module -never requires the management SDKs or a live Azure session. The CLI layer in -:mod:`agentops.cli.app` stays thin and delegates the Azure logic here. -""" - -from __future__ import annotations - -import json -import logging -import shutil -import subprocess -import tempfile -import uuid -from dataclasses import dataclass, field -from importlib.resources import files as _pkg_files -from pathlib import Path -from typing import Any, Dict, Iterable, List, Mapping, Optional - -log = logging.getLogger(__name__) - -_TEMPLATE_PACKAGE = "agentops.templates" -_WORKBOOK_RESOURCE_PATH = "workbooks/foundry-ops.workbook.json" - -#: ARM resource type for an Azure Monitor workbook. -WORKBOOK_RESOURCE_TYPE = "Microsoft.Insights/workbooks" - -#: Diagnostic log categories the Azure OpenAI resource must emit for the -#: workbook queries to return data. -REQUIRED_DIAGNOSTIC_CATEGORIES = ("RequestResponse", "AzureOpenAIRequestUsage") - -_PORTAL_BASE = "https://portal.azure.com" - -# Deterministic namespace so ``deploy`` and ``open`` agree on the workbook -# resource name without querying Azure. -_WORKBOOK_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_DNS, "agentops.foundry-ops.workbook") - -# Built-in Azure role definition GUIDs used by the preflight. A role that can -# write workbooks (Owner / Contributor / Workbook Contributor) satisfies the -# deploy requirement; a role that can read the workspace satisfies the query -# requirement. -_ROLE_WORKBOOK_CONTRIBUTOR = "e8ddcd69-c73f-4f9f-9844-4100522f16ad" -_ROLE_LOG_ANALYTICS_READER = "73c42c96-874c-492b-b04d-ab87d138a893" -_ROLE_LOG_ANALYTICS_CONTRIBUTOR = "92aaf0da-9dab-42b6-94a3-d43ce8d16293" -_ROLE_READER = "acdd72a7-3385-48ef-bd42-f606fba81ae7" -_ROLE_CONTRIBUTOR = "b24988ac-6180-42a0-ab88-20f7382dd24c" -_ROLE_OWNER = "8e3af657-a8ff-443c-a75c-2fe8c4bcb635" - -_WORKBOOK_WRITE_ROLES = frozenset( - {_ROLE_WORKBOOK_CONTRIBUTOR, _ROLE_CONTRIBUTOR, _ROLE_OWNER} -) -_WORKSPACE_READ_ROLES = frozenset( - { - _ROLE_LOG_ANALYTICS_READER, - _ROLE_LOG_ANALYTICS_CONTRIBUTOR, - _ROLE_READER, - _ROLE_CONTRIBUTOR, - _ROLE_OWNER, - } -) - - -class DashboardError(RuntimeError): - """Raised for user-facing dashboard failures (surfaced by the CLI).""" - - -@dataclass -class DashboardTarget: - """Resolved Azure context for the workbook deploy.""" - - name: str = "AgentOps Foundry operations" - subscription_id: Optional[str] = None - resource_group: Optional[str] = None - workspace_id: Optional[str] = None - aoai_resource_id: Optional[str] = None - aoai_account_name: Optional[str] = None - tenant_id: Optional[str] = None - location: Optional[str] = None - discovery: Dict[str, Any] = field(default_factory=dict) - - -@dataclass -class PreflightResult: - """Outcome of a single preflight check. - - ``ok`` is ``True`` when the check passed. ``level`` is ``"ok"``, - ``"warn"`` (deploy can still proceed) or ``"error"`` (deploy must stop). - ``messages`` carries friendly, actionable text for the CLI to print. - """ - - ok: bool - level: str - messages: List[str] = field(default_factory=list) - - -# --------------------------------------------------------------------------- -# Template loading -# --------------------------------------------------------------------------- -def load_workbook_template() -> str: - """Return the packaged workbook JSON as a string.""" - resource = _pkg_files(_TEMPLATE_PACKAGE).joinpath(_WORKBOOK_RESOURCE_PATH) - return resource.read_text(encoding="utf-8") - - -def load_workbook_content() -> Dict[str, Any]: - """Return the packaged workbook JSON parsed into a dict.""" - try: - return json.loads(load_workbook_template()) - except (OSError, json.JSONDecodeError) as exc: # pragma: no cover - packaging - raise DashboardError( - f"Could not load the packaged workbook template: {exc}" - ) from exc - - -# --------------------------------------------------------------------------- -# Portal URL -# --------------------------------------------------------------------------- -def make_workbook_resource_id( - subscription_id: str, resource_group: str, name: str -) -> str: - """Build the deterministic ARM id for the deployed workbook. - - The workbook resource name is a GUID derived from the display name and - resource group so ``deploy`` and ``open`` agree without a live lookup. - """ - guid = str(uuid.uuid5(_WORKBOOK_NAMESPACE, f"{resource_group}:{name}")) - return ( - f"/subscriptions/{subscription_id}" - f"/resourceGroups/{resource_group}" - f"/providers/{WORKBOOK_RESOURCE_TYPE}/{guid}" - ) - - -def build_workbook_portal_url( - *, - workbook_resource_id: Optional[str] = None, - tenant_id: Optional[str] = None, - subscription_id: Optional[str] = None, - resource_group: Optional[str] = None, - name: Optional[str] = None, -) -> str: - """Return an Azure portal deep link for the workbook. - - When the workbook ARM id is known (directly, or derivable from - subscription/resource-group/name) the link opens that workbook. Otherwise - it falls back to the Azure Monitor Workbooks gallery so the tile is never - broken. - """ - rid = workbook_resource_id - if not rid and subscription_id and resource_group and name: - rid = make_workbook_resource_id(subscription_id, resource_group, name) - if rid: - prefix = f"{_PORTAL_BASE}/#@{tenant_id}" if tenant_id else f"{_PORTAL_BASE}/#" - return f"{prefix}/resource{rid}/workbook" - return ( - f"{_PORTAL_BASE}/#view/Microsoft_Azure_Monitoring_Workbooks" - "/WorkbookMenuBlade/~/gallery" - ) - - -# --------------------------------------------------------------------------- -# Diagnostic settings -# --------------------------------------------------------------------------- -def missing_diagnostic_categories(enabled_categories: Iterable[str]) -> List[str]: - """Return the required categories not present in ``enabled_categories``.""" - enabled = {str(c) for c in enabled_categories} - return [c for c in REQUIRED_DIAGNOSTIC_CATEGORIES if c not in enabled] - - -def build_diagnostic_settings_command( - *, - aoai_resource_id: Optional[str], - workspace_id: Optional[str], - name: str = "agentops-foundry-ops", -) -> str: - """Return the exact ``az`` command that wires the required categories.""" - resource = aoai_resource_id or "" - workspace = workspace_id or "" - logs = json.dumps( - [{"category": c, "enabled": True} for c in REQUIRED_DIAGNOSTIC_CATEGORIES] - ) - return ( - "az monitor diagnostic-settings create " - f"--name {name} " - f"--resource {resource} " - f"--workspace {workspace} " - f"--logs '{logs}'" - ) - - -# --------------------------------------------------------------------------- -# RBAC preflight -# --------------------------------------------------------------------------- -def check_rbac( - *, - subscription_id: Optional[str], - resource_group: Optional[str], - workspace_id: Optional[str], -) -> PreflightResult: - """Preflight the caller's RBAC for a workbook deploy. - - Requires ``Workbook Contributor`` (or a superset) on the resource group and - ``Log Analytics Reader`` (or a superset) on the workspace. When the RBAC - listing cannot run (SDK missing, no credential, listing denied) the check - fails **open** with a warning so ``deploy`` can still be attempted and let - ARM enforce permissions. - """ - if not subscription_id or not resource_group: - return PreflightResult( - ok=False, - level="error", - messages=[ - "Cannot check RBAC: subscription and resource group are " - "unknown. Pass --subscription and --resource-group, or run " - "from an initialized AgentOps workspace.", - ], - ) - - rg_scope = f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}" - try: - from agentops.agent.checks._rbac_authorization import ( - AuthorizationCheckError, - list_principal_role_definition_ids, - resolve_signed_in_principal_object_id, - ) - except ImportError as exc: # pragma: no cover - shipped together - return PreflightResult( - ok=True, - level="warn", - messages=[ - "Skipping RBAC preflight (authorization helpers unavailable: " - f"{exc}). ARM will enforce permissions at deploy time.", - ], - ) - - try: - principal = resolve_signed_in_principal_object_id() - rg_roles = set( - list_principal_role_definition_ids( - subscription_id=subscription_id, - scope=rg_scope, - principal_object_id=principal, - ) - ) - except AuthorizationCheckError as exc: - return PreflightResult( - ok=True, - level="warn", - messages=[ - f"Skipping RBAC preflight ({exc}). ARM will enforce " - "permissions at deploy time.", - ], - ) - - messages: List[str] = [] - ok = True - if not (rg_roles & _WORKBOOK_WRITE_ROLES): - ok = False - messages.append( - "You need the 'Workbook Contributor' role on resource group " - f"'{resource_group}'. Ask an admin to grant it, or run with " - "--dry-run to emit the ARM template instead." - ) - - if workspace_id: - try: - ws_roles = set( - list_principal_role_definition_ids( - subscription_id=subscription_id, - scope=workspace_id, - principal_object_id=principal, - ) - ) - except AuthorizationCheckError as exc: - messages.append( - f"Could not verify Log Analytics access ({exc}); make sure you " - "have 'Log Analytics Reader' on the workspace." - ) - ws_roles = set() - if ws_roles and not (ws_roles & _WORKSPACE_READ_ROLES): - ok = False - messages.append( - "You need the 'Log Analytics Reader' role on workspace " - f"'{workspace_id}' so the workbook can query it." - ) - - if ok and not messages: - messages.append( - "RBAC preflight passed: you can write workbooks to " - f"'{resource_group}' and read the workspace." - ) - return PreflightResult(ok=ok, level="ok" if ok else "error", messages=messages) - - -# --------------------------------------------------------------------------- -# Discovery -# --------------------------------------------------------------------------- -_WORKSPACE_ENV_KEYS = ( - "AZURE_LOG_ANALYTICS_WORKSPACE_ID", - "AZURE_LOG_ANALYTICS_WORKSPACE_RESOURCE_ID", - "LOG_ANALYTICS_WORKSPACE_ID", - "AZURE_MONITOR_WORKSPACE_ID", - "AZURE_MONITOR_WORKSPACE_RESOURCE_ID", -) -_ACCOUNT_ENV_KEYS = ( - "AZURE_OPENAI_RESOURCE", - "AZURE_OPENAI_RESOURCE_NAME", - "AZURE_AI_SERVICES_RESOURCE_NAME", - "AZURE_AI_SERVICES_NAME", -) -_TENANT_ENV_KEYS = ("AZURE_TENANT_ID",) - - -def _first(values: Mapping[str, str], keys: Iterable[str]) -> Optional[str]: - for key in keys: - val = values.get(key) - if val: - return val - return None - - -def discover_target( - workspace: Optional[Path], - *, - subscription_id: Optional[str] = None, - resource_group: Optional[str] = None, - workspace_id: Optional[str] = None, - name: Optional[str] = None, -) -> DashboardTarget: - """Resolve the deploy target from explicit flags, then the AZD env. - - Reuses the AgentOps foundry discovery path (the AZD ``.env`` read used by - the Azure resources doctor source). Explicit arguments always win. - """ - env_values: Dict[str, str] = {} - discovery: Dict[str, Any] = {} - try: - from agentops.agent.sources.azure_resources import ( - _discover_azd_environment, - ) - - _env_name, env_values, azd_diag = _discover_azd_environment(workspace) - discovery["azd"] = azd_diag - except Exception as exc: # noqa: BLE001 - discovery is best-effort - discovery["azd"] = {"status": "error", "reason": str(exc)} - - sub = subscription_id or env_values.get("AZURE_SUBSCRIPTION_ID") - rg = resource_group or env_values.get("AZURE_RESOURCE_GROUP") - ws = workspace_id or _first(env_values, _WORKSPACE_ENV_KEYS) - tenant = _first(env_values, _TENANT_ENV_KEYS) - account = _first(env_values, _ACCOUNT_ENV_KEYS) - - aoai_resource_id: Optional[str] = None - if account and account.startswith("/subscriptions/"): - aoai_resource_id = account - account = account.rstrip("/").rsplit("/", 1)[-1] - elif account and sub and rg: - aoai_resource_id = ( - f"/subscriptions/{sub}/resourceGroups/{rg}" - f"/providers/Microsoft.CognitiveServices/accounts/{account}" - ) - - return DashboardTarget( - name=name or "AgentOps Foundry operations", - subscription_id=sub, - resource_group=rg, - workspace_id=ws, - aoai_resource_id=aoai_resource_id, - aoai_account_name=account, - tenant_id=tenant, - discovery=discovery, - ) - - -# --------------------------------------------------------------------------- -# ARM template + deploy -# --------------------------------------------------------------------------- -def build_arm_template( - *, - target: DashboardTarget, - workbook_content: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """Wrap the workbook content into a deployable ARM template.""" - content = ( - workbook_content if workbook_content is not None else load_workbook_content() - ) - guid = str( - uuid.uuid5(_WORKBOOK_NAMESPACE, f"{target.resource_group}:{target.name}") - ) - resource: Dict[str, Any] = { - "type": WORKBOOK_RESOURCE_TYPE, - "apiVersion": "2022-04-01", - "name": guid, - # Workbooks reject "global"; resolve to the target region at deploy - # time. The RG-scoped deployment always has a valid location, and an - # explicit target.location (a real Azure region) still wins when set. - "location": target.location or "[resourceGroup().location]", - "kind": "shared", - "properties": { - "displayName": target.name, - "serializedData": json.dumps(content), - "version": "1.0", - "sourceId": target.workspace_id or "Azure Monitor", - "category": "workbook", - }, - } - return { - "$schema": ( - "https://schema.management.azure.com/schemas/2019-04-01/" - "deploymentTemplate.json#" - ), - "contentVersion": "1.0.0.0", - "resources": [resource], - "outputs": { - "workbookId": { - "type": "string", - "value": f"[resourceId('{WORKBOOK_RESOURCE_TYPE}', '{guid}')]", - } - }, - } - - -def _az_executable() -> Optional[str]: - return shutil.which("az") or shutil.which("az.cmd") - - -def deploy_workbook( - *, - target: DashboardTarget, - workbook_content: Optional[Dict[str, Any]] = None, - timeout: int = 300, -) -> str: - """Deploy the workbook via ``az deployment group create``. - - Returns the portal URL for the deployed workbook. Raises - :class:`DashboardError` with a friendly message on any failure. - """ - if not target.subscription_id or not target.resource_group: - raise DashboardError( - "Deploy needs a subscription and resource group. Pass " - "--subscription and --resource-group, or run from an initialized " - "AgentOps workspace." - ) - az = _az_executable() - if az is None: - raise DashboardError( - "The Azure CLI ('az') was not found on PATH. Install it and run " - "'az login', or use --dry-run to emit the ARM template." - ) - - template = build_arm_template(target=target, workbook_content=workbook_content) - with tempfile.TemporaryDirectory(prefix="agentops-workbook-") as tmp: - template_path = Path(tmp) / "foundry-ops.deploy.json" - template_path.write_text(json.dumps(template), encoding="utf-8") - cmd = [ - az, - "deployment", - "group", - "create", - "--subscription", - target.subscription_id, - "--resource-group", - target.resource_group, - "--name", - "agentops-foundry-ops", - "--template-file", - str(template_path), - "--only-show-errors", - "--output", - "json", - ] - try: - completed = subprocess.run( # noqa: S603 - args are controlled - cmd, - capture_output=True, - text=True, - timeout=timeout, - check=False, - ) - except (OSError, subprocess.TimeoutExpired) as exc: - raise DashboardError( - f"Failed to run 'az deployment group create': {exc}" - ) from exc - - if completed.returncode != 0: - detail = (completed.stderr or completed.stdout or "").strip() - raise DashboardError( - f"Workbook deployment failed. The Azure CLI reported:\n{detail}" - ) - - return build_workbook_portal_url( - subscription_id=target.subscription_id, - resource_group=target.resource_group, - name=target.name, - tenant_id=target.tenant_id, - ) diff --git a/src/agentops/services/telemetry_import.py b/src/agentops/services/telemetry_import.py deleted file mode 100644 index 286778c..0000000 --- a/src/agentops/services/telemetry_import.py +++ /dev/null @@ -1,550 +0,0 @@ -"""Import Azure Monitor telemetry into AgentOps JSONL datasets. - -The module has two halves: - -* a pure transformer that maps telemetry rows into AgentOps dataset rows -* a thin Azure Monitor query wrapper with lazy SDK imports - -Users never provide raw KQL. The query builder only accepts structured time -ranges, field mappings, filters, and row limits from ``agentops.yaml``. -""" - -from __future__ import annotations - -import json -import os -import re -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Iterable, Optional - -from agentops.core.agentops_config import AgentOpsConfig, TelemetryImportConfig - -DEFAULT_MAX_ROWS = 100 -MAX_ROWS_CAP = 5000 - -_DEFAULT_FIELD_CANDIDATES: dict[str, tuple[str, ...]] = { - "input": ( - "input", - "query", - "prompt", - "message", - "user_message", - "customDimensions.input", - "customDimensions.query", - "customDimensions.prompt", - "customDimensions.gen_ai.prompt", - ), - "response": ( - "response", - "prediction", - "output", - "answer", - "completion", - "assistant_message", - "customDimensions.response", - "customDimensions.prediction", - "customDimensions.output", - "customDimensions.gen_ai.completion", - ), - "context": ( - "context", - "retrieved_context", - "grounding", - "customDimensions.context", - "customDimensions.retrieved_context", - "customDimensions.grounding", - ), - "retrieved_context_items": ( - "retrieved_context_items", - "context_items", - "customDimensions.retrieved_context_items", - "customDimensions.context_items", - ), - "tool_calls": ("tool_calls", "customDimensions.tool_calls"), - "trace_id": ("trace_id", "operation_Id", "operationId"), - "turn_id": ("turn_id", "span_id", "id", "customDimensions.turn_id"), - "timestamp": ("timestamp", "TimeGenerated", "time"), -} - -_QUERY_COLUMNS = ( - "timestamp", - "operation_Id = column_ifexists('operation_Id', '')", - "operationId = column_ifexists('operationId', '')", - "id = column_ifexists('id', '')", - "name = column_ifexists('name', '')", - "message = column_ifexists('message', '')", - "duration = column_ifexists('duration', '')", - "success = column_ifexists('success', '')", - "customDimensions = column_ifexists('customDimensions', dynamic({}))", -) - - -class TelemetryImportError(RuntimeError): - """Raised when a telemetry import cannot be validated, queried, or written.""" - - -@dataclass(frozen=True) -class TelemetryImportPreview: - """Result of validating/querying/transforming one telemetry import.""" - - config: TelemetryImportConfig - output_path: Path - manifest_path: Path - rows: list[dict[str, Any]] - skipped: int = 0 - deduped: int = 0 - truncated: bool = False - warnings: list[str] = field(default_factory=list) - - -def find_telemetry_import( - config: AgentOpsConfig, - name: str, -) -> TelemetryImportConfig: - """Return a named telemetry import or raise a friendly error.""" - - for item in config.telemetry_imports: - if item.name == name: - return item - available = ", ".join(item.name for item in config.telemetry_imports) or "none" - raise TelemetryImportError( - f"telemetry import {name!r} was not found in agentops.yaml. " - f"Available imports: {available}." - ) - - -def validate_telemetry_import(config: TelemetryImportConfig) -> list[str]: - """Validate service-level constraints and return non-fatal warnings.""" - - warnings: list[str] = [] - if config.output.label_mode == "self-similarity": - warnings.append( - "Generated rows use production responses as expected values for drift " - "detection, not human-verified ground truth." - ) - return warnings - - -def preview_telemetry_import( - config: TelemetryImportConfig, - *, - rows: Optional[int] = None, - apply: bool = False, -) -> TelemetryImportPreview: - """Query Azure Monitor, transform rows, and optionally write JSONL output.""" - - validate_telemetry_import(config) - raw_rows = query_azure_monitor(config, rows=rows) - preview = transform_telemetry_rows(config, raw_rows, rows=rows) - if apply: - write_telemetry_import(preview) - return preview - - -def transform_telemetry_rows( - config: TelemetryImportConfig, - telemetry_rows: Iterable[dict[str, Any]], - *, - rows: Optional[int] = None, -) -> TelemetryImportPreview: - """Pure transformation from telemetry records to AgentOps dataset rows.""" - - limit = _bounded_rows(rows if rows is not None else config.max_rows) - output_path = config.output.path - manifest_path = config.output.manifest_path or output_path.with_name( - f"{output_path.stem}-manifest.json" - ) - warnings = validate_telemetry_import(config) - converted: list[dict[str, Any]] = [] - skipped = 0 - deduped = 0 - seen: set[tuple[str, str]] = set() - - for raw in telemetry_rows: - if len(converted) >= limit: - break - row = _telemetry_row_to_agentops_row(config, raw) - if row is None: - skipped += 1 - continue - telemetry = row.get("telemetry") - trace_id = "" - turn_id = "" - if isinstance(telemetry, dict): - trace_id = str(telemetry.get("trace_id") or "") - turn_id = str(telemetry.get("turn_id") or "") - key = (trace_id or row["input"], turn_id or row.get("response", "")) - if key in seen: - deduped += 1 - continue - seen.add(key) - converted.append(row) - - truncated = len(converted) >= limit - if not converted: - warnings.append("No telemetry rows contained both input and response text.") - return TelemetryImportPreview( - config=config, - output_path=output_path, - manifest_path=manifest_path, - rows=converted, - skipped=skipped, - deduped=deduped, - truncated=truncated, - warnings=warnings, - ) - - -def write_telemetry_import(preview: TelemetryImportPreview) -> None: - """Write JSONL rows and a small manifest next to the output.""" - - preview.output_path.parent.mkdir(parents=True, exist_ok=True) - with preview.output_path.open("w", encoding="utf-8") as handle: - for row in preview.rows: - handle.write(json.dumps(row, ensure_ascii=False) + "\n") - - trace_ids = [ - str(row.get("telemetry", {}).get("trace_id")) - for row in preview.rows - if isinstance(row.get("telemetry"), dict) and row["telemetry"].get("trace_id") - ] - manifest = { - "version": 1, - "generated_at": datetime.now(timezone.utc).isoformat(), - "import": preview.config.name, - "source": preview.config.source, - "target": preview.config.target, - "output_path": str(preview.output_path), - "rows": len(preview.rows), - "skipped": preview.skipped, - "deduped": preview.deduped, - "truncated": preview.truncated, - "trace_ids": trace_ids, - "warnings": preview.warnings, - } - preview.manifest_path.write_text( - json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - - -def render_telemetry_import_preview(preview: TelemetryImportPreview) -> str: - """Render concise CLI output.""" - - lines = [ - "AgentOps telemetry import", - f"Import: {preview.config.name}", - f"Target: {preview.config.target}", - f"Output: {preview.output_path}", - "", - "Summary", - f" rows {len(preview.rows)}", - f" skipped {preview.skipped}", - f" deduped {preview.deduped}", - f" truncated {str(preview.truncated).lower()}", - ] - if preview.warnings: - lines.append("") - lines.append("Warnings") - lines.extend(f" - {warning}" for warning in preview.warnings) - if preview.rows: - lines.append("") - lines.append("Sample rows") - for index, row in enumerate(preview.rows[:3], start=1): - lines.append(f" {index}. {str(row.get('input', ''))[:120]}") - return "\n".join(lines) + "\n" - - -def query_azure_monitor( - config: TelemetryImportConfig, - *, - rows: Optional[int] = None, -) -> list[dict[str, Any]]: - """Run the generated KQL against Azure Monitor with lazy SDK imports.""" - - try: - from azure.identity import DefaultAzureCredential # noqa: WPS433 - except ImportError as exc: - raise TelemetryImportError( - "Telemetry import requires Azure authentication packages. Install " - "them with: python -m pip install azure-identity azure-monitor-query" - ) from exc - - kql = build_telemetry_kql(config, rows=rows) - credential = DefaultAzureCredential( - exclude_developer_cli_credential=True, - process_timeout=30, - ) - try: - if config.target == "log-analytics": - from azure.monitor.query import LogsQueryClient # noqa: WPS433 - - client = LogsQueryClient(credential) - workspace_id = _resolve_value(config.workspace_id, "workspace_id") - response = client.query_workspace(workspace_id, kql, timespan=None) - return _flatten_logs_response(response) - if config.resource_id: - from azure.monitor.query import LogsQueryClient # noqa: WPS433 - - client = LogsQueryClient(credential) - resource_id = _resolve_value(config.resource_id, "resource_id") - response = client.query_resource(resource_id, kql, timespan=None) - return _flatten_logs_response(response) - app_id = _application_id(config) - token = credential.get_token("https://api.applicationinsights.io/.default").token - return _query_application_insights(app_id, token, kql) - except ImportError as exc: - raise TelemetryImportError( - "Telemetry import with resource_id/workspace_id requires the Azure " - "Monitor Query SDK. Install it with: python -m pip install " - "azure-monitor-query" - ) from exc - except TelemetryImportError: - raise - except Exception as exc: # noqa: BLE001 - raise TelemetryImportError(f"Azure Monitor query failed: {exc}") from exc - - -def build_telemetry_kql( - config: TelemetryImportConfig, - *, - rows: Optional[int] = None, -) -> str: - """Build safe KQL from structured config only.""" - - limit = _bounded_rows(rows if rows is not None else config.max_rows) - clauses = ["union isfuzzy=true requests, dependencies, traces"] - clauses.append(f"| extend timestamp = {_timestamp_expr()}") - clauses.append(_time_clause(config)) - for key, value in sorted(config.filters.items()): - clauses.append(_filter_clause(key, value)) - columns = ", ".join(_QUERY_COLUMNS) - clauses.append(f"| project {columns}") - clauses.append("| order by timestamp desc") - clauses.append(f"| take {limit}") - return "\n".join(clauses) - - -def _telemetry_row_to_agentops_row( - config: TelemetryImportConfig, - raw: dict[str, Any], -) -> Optional[dict[str, Any]]: - input_text = _mapped_text(config, raw, "input") - response_text = _mapped_text(config, raw, "response") - if not input_text or not response_text: - return None - - label_mode = config.output.label_mode - telemetry = { - "trace_id": _mapped_text(config, raw, "trace_id"), - "turn_id": _mapped_text(config, raw, "turn_id"), - "timestamp": _mapped_text(config, raw, "timestamp"), - "source": config.source, - "target": config.target, - "import": config.name, - } - row: dict[str, Any] = { - "input": _clean_value(input_text, config), - "response": _clean_value(response_text, config), - "prediction": _clean_value(response_text, config), - "expected": _clean_value(response_text, config) if label_mode == "self-similarity" else "", - "telemetry": {k: v for k, v in telemetry.items() if v not in (None, "")}, - "metadata": { - "source": "azure_monitor_telemetry", - "label_mode": label_mode, - "needs_review": True, - }, - } - context = _mapped_value(config, raw, "context") - if context not in (None, "", [], {}): - row["context"] = _clean_value(context, config) - row["retrieved_context"] = row["context"] - context_items = _mapped_value(config, raw, "retrieved_context_items") - if context_items not in (None, "", [], {}): - row["retrieved_context_items"] = _clean_value(context_items, config) - tool_calls = _mapped_value(config, raw, "tool_calls") - if tool_calls not in (None, "", [], {}): - row["tool_calls"] = _clean_value(tool_calls, config) - if config.privacy.include_raw: - row["raw"] = _clean_value(raw, config) - return row - - -def _mapped_text(config: TelemetryImportConfig, raw: dict[str, Any], name: str) -> Optional[str]: - value = _mapped_value(config, raw, name) - if value is None: - return None - if isinstance(value, str): - value = value.strip() - return value or None - if isinstance(value, (dict, list)): - text = json.dumps(value, ensure_ascii=False) - return text if text not in ("{}", "[]") else None - text = str(value).strip() - return text or None - - -def _mapped_value(config: TelemetryImportConfig, raw: dict[str, Any], name: str) -> Any: - mapping = config.fields.get(name) - if mapping: - return _lookup(raw, mapping) - for candidate in _DEFAULT_FIELD_CANDIDATES.get(name, ()): - value = _lookup(raw, candidate) - if value not in (None, "", [], {}): - return value - return None - - -def _lookup(data: dict[str, Any], path: str) -> Any: - current: Any = data - for part in path.split("."): - if not isinstance(current, dict): - return None - current = current.get(part) - return current - - -def _clean_value(value: Any, config: TelemetryImportConfig, key: str = "") -> Any: - lowered = key.lower() - if any(fragment.lower() in lowered for fragment in config.privacy.redact_fields): - return "[redacted]" - if isinstance(value, dict): - return {k: _clean_value(v, config, str(k)) for k, v in value.items()} - if isinstance(value, list): - return [_clean_value(item, config, key) for item in value] - if isinstance(value, str) and len(value) > config.privacy.max_field_length: - return value[: config.privacy.max_field_length] + "...[truncated]" - return value - - -def _flatten_logs_response(response: Any) -> list[dict[str, Any]]: - tables = getattr(response, "tables", None) or [] - if not tables: - return [] - table = tables[0] - columns: list[str] = [] - for column in getattr(table, "columns", None) or []: - name = getattr(column, "name", None) if not isinstance(column, dict) else column.get("name") - if isinstance(name, str): - columns.append(name) - rows: list[dict[str, Any]] = [] - for raw in getattr(table, "rows", None) or []: - rows.append(dict(zip(columns, raw))) - return rows - - -def _application_id(config: TelemetryImportConfig) -> str: - if config.application_id: - return _resolve_value(config.application_id, "application_id") - if config.connection_string: - connection_string = _resolve_value(config.connection_string, "connection_string") - match = re.search(r"ApplicationId=([0-9a-fA-F-]+)", connection_string) - if match: - return match.group(1) - raise TelemetryImportError( - "application-insights imports require resource_id, application_id, or " - "a connection_string containing ApplicationId" - ) - - -def _query_application_insights(app_id: str, bearer: str, kql: str) -> list[dict[str, Any]]: - import json as _json - from urllib import request - - body = _json.dumps({"query": kql}).encode("utf-8") - req = request.Request( - url=f"https://api.applicationinsights.io/v1/apps/{app_id}/query", - data=body, - headers={ - "Authorization": f"Bearer {bearer}", - "Content-Type": "application/json", - }, - method="POST", - ) - with request.urlopen(req, timeout=30) as response: # noqa: S310 - parsed = _json.loads(response.read()) - if isinstance(parsed, dict) and parsed.get("error"): - err = parsed["error"] - message = err.get("message") if isinstance(err, dict) else str(err) - raise TelemetryImportError(f"Application Insights query failed: {message}") - tables = parsed.get("tables") if isinstance(parsed, dict) else None - if not tables: - return [] - table = tables[0] - columns = [column.get("name") for column in table.get("columns", [])] - return [dict(zip(columns, row)) for row in table.get("rows", [])] - - -def _time_clause(config: TelemetryImportConfig) -> str: - tr = config.time_range - if tr.from_ and tr.to: - return ( - f"| where timestamp between (datetime({_kql_string(tr.from_)}) .. " - f"datetime({_kql_string(tr.to)}))" - ) - days = tr.lookback_days or 7 - return f"| where timestamp >= ago({days}d)" - - -def _filter_clause(key: str, value: str | list[str]) -> str: - expr = _safe_column_expr(key) - values = value if isinstance(value, list) else [value] - escaped = ", ".join(_kql_string(str(item)) for item in values) - if len(values) == 1: - return f"| where {expr} == {escaped}" - return f"| where {expr} in ({escaped})" - - -def _safe_column_expr(key: str) -> str: - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?", key): - raise TelemetryImportError( - f"unsafe telemetry filter field {key!r}; use a column name or customDimensions.name" - ) - if key.startswith("customDimensions."): - subkey = key.split(".", 1)[1] - return ( - "tostring(column_ifexists('customDimensions', dynamic({}))" - f"[{_kql_string(subkey)}])" - ) - return f"tostring(column_ifexists({_kql_string(key)}, ''))" - - -def _timestamp_expr() -> str: - return ( - "coalesce(" - "column_ifexists('timestamp', datetime(null)), " - "column_ifexists('TimeGenerated', datetime(null)), " - "column_ifexists('time', datetime(null))" - ")" - ) - - -def _kql_string(value: str) -> str: - return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" - - -def _resolve_value(value: Optional[str], label: str) -> str: - if not value: - raise TelemetryImportError(f"telemetry import is missing {label}") - value = value.strip() - env_name: Optional[str] = None - if value.startswith("env:"): - env_name = value[4:] - elif value.startswith("$") and len(value) > 1: - env_name = value[1:].strip("{}") - if env_name: - resolved = os.getenv(env_name) - if not resolved: - raise TelemetryImportError( - f"environment variable {env_name} referenced by {label} is not set" - ) - return resolved - return value - - -def _bounded_rows(rows: int) -> int: - if rows <= 0: - raise TelemetryImportError("rows must be greater than zero") - return min(rows, MAX_ROWS_CAP) diff --git a/src/agentops/templates/workbooks/README.md b/src/agentops/templates/workbooks/README.md deleted file mode 100644 index 5ae54f5..0000000 --- a/src/agentops/templates/workbooks/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# Foundry operations dashboard (Azure Monitor workbook) - -`foundry-ops.workbook.json` is an Azure Monitor **gallery-template workbook** that -visualizes Azure AI Foundry / Azure OpenAI operational metrics end to end: PTU -utilization, PAYG spillover, traffic and token consumption, latency percentiles, -error / throttling rates, and Foundry-owned trace-evaluation results. - -The workbook is scoped **per Azure OpenAI resource** and **per Log Analytics -workspace**. Pick the subscription, workspace, and Azure OpenAI resource in the -parameters bar, then use the deployment, model, time-range, and streaming -filters to narrow each tab. - -## Tabs - -| Tab | Shows | -| --- | --- | -| Capacity | `PTU_Avg_Pct`, `PTU_Max_Pct`, `Ratelimit`, PAYG `Spillover`, `Aggregated_PTU_With_Spillover`, `PTU_Normalizado` | -| Traffic and tokens | `TotalCalls`, `AzureOpenAIRequests`, `InputTokens`, `OutputTokens`, `TotalTokens`, `ProcessedPromptTokens`, `GeneratedTokens`, `TokensPorRequest`, and a PTU vs PAYG spillover pivot | -| Latency | `TTFT`, `TBT`, `TTLT`, tokens/sec, time-to-response, and `P95` / `P99` / `Avg` / `Max` over `DurationMs` | -| Errors and throttling | HTTP `429` / `400` / `500` counts, `BlockedCalls`, `TasaThrottling_Pct`, `TasaError_Pct` | -| Agent behavior | Data status and freshness, observed `invoke_agent` invocations, evaluated traces, evaluation-event counts, per-evaluator pass rate / volume / raw-score trends, and recent failed or low-score trace correlation | - -`PTU_Normalizado` (`PTU% * 100000`) is a **display-only** scaling series so the -0-100 PTU utilization percentage can share a Y axis with raw token counts in a -single timechart. Read true utilization from `PTU_Avg_Pct` / `PTU_Max_Pct`. - -## KQL queries - -The raw KQL for each derived-metric family lives under -[`queries/`](./queries), so operators can reuse the logic outside the workbook: - -| File | Derived metrics | -| --- | --- | -| [`capacity_ptu_spillover.kql`](./queries/capacity_ptu_spillover.kql) | PTU avg/max %, Ratelimit, Spillover, Aggregated PTU with spillover, PTU_Normalizado | -| [`traffic_tokens.kql`](./queries/traffic_tokens.kql) | Calls, input/output/total tokens, processed/generated tokens, tokens per request, PTU vs PAYG split | -| [`latency_percentiles.kql`](./queries/latency_percentiles.kql) | TTFT/TBT/TTLT, tokens/sec, time-to-response, and P95/P99/Avg/Max over DurationMs | -| [`errors_throttling.kql`](./queries/errors_throttling.kql) | HTTP 429/400/500 counts, blocked calls, throttling rate, error rate | -| [`agent_behavior.kql`](./queries/agent_behavior.kql) | Versioned `agent_behavior/v1` normalization for Foundry evaluation events and observed `invoke_agent` spans across workspace-based and classic Application Insights tables | - -The standalone `.kql` files expose the parameter bindings as `let` declarations -at the top; the workbook substitutes the same values through its parameter bar. - -## Agent behavior tab - -The **Agent behavior** tab is an additive, read-only adapter over compatible -`gen_ai.evaluation.result` events. Current official Foundry documentation -explicitly verifies this event for -[human trace annotations](https://learn.microsoft.com/azure/foundry/observability/how-to/trace-annotations#log-end-user-feedback-as-trace-annotations). -It does not clearly guarantee that every automated Foundry trace-evaluation -result is exported through the same Application Insights event shape. -Automated event support is therefore **validation-dependent** and must be -proven in the target workspace. Trace evaluation and annotations are preview, -platform-owned features. AgentOps does not create, schedule, gate, or edit -them. - -The `agent_behavior/v1` KQL normalizer supports both recognized Application -Insights shapes: - -- workspace-based `AppEvents` (`Name`, `Properties`) and `AppDependencies` - (`Properties`); -- classic `customEvents` (`name`, `customDimensions`) and `dependencies` - (`customDimensions`). - -The mapping uses an explicit set of known `gen_ai.*` properties and keeps the -raw property bag in the normalized row. Missing optional values do not remove -events. Missing agent versions appear as **Version not reported**, and -nonnumeric score values remain visible as raw score text rather than becoming -zero. Trace-ID evaluation and correlation do not require `gen_ai.agent.id`; -agent and version filters simply use the metadata when a producer reports it. - -The status row appears before quality results and distinguishes these states: - -| State | Meaning | -| --- | --- | -| `Schema unavailable` | Matching event names exist, but none of the v1 evaluator, score, or label properties are recognized. Inspect the retained raw properties before updating the versioned mapping. | -| `No access` | The workbook query shows a native permission error. Request `Log Analytics Reader` on the selected workspace. | -| `No data` | The recognized event tables contain no compatible `gen_ai.evaluation.result` events in the selected time range. This can also mean automated trace evaluation does not export this event shape in the target environment. | -| `Filter empty` | Events exist in the time range, but none match the environment, agent, version, and evaluator filters. | -| `Possible ingestion delay` | The newest matching event is more than 15 minutes old. This may be ingestion delay or simply no recent Foundry trace evaluation. | - -The three count columns intentionally have different meanings: - -- **Observed invoke_agent invocations** counts distinct invocation keys visible - in the selected workspace. -- **Evaluated traces** counts distinct trace IDs represented by evaluation - events. -- **Evaluation events** counts evaluator results. One evaluated trace can emit - several events. - -The workbook does not calculate evaluation coverage because the workspace might -not contain a complete invocation denominator. Pass rate uses only recognized -pass/fail labels. Raw score trends stay in a table grouped by evaluator because -different evaluators can use different scales; pass-rate and event-volume charts -use comparable units. - -The recent-events table keeps trace, response, and conversation IDs when -available. Copy the trace ID, open the matching Microsoft Foundry project, -select **Tracing**, and search for the ID. The workbook cannot build a reliable -project-specific Foundry deep link from a Log Analytics workspace ID alone. - -## Required diagnostic settings - -The Azure OpenAI resource must send both of these log categories to the Log -Analytics workspace the workbook queries: - -- `RequestResponse` -- `AzureOpenAIRequestUsage` - -If they are missing, create them with: - -```bash -az monitor diagnostic-settings create \ - --name agentops-foundry-ops \ - --resource \ - --workspace \ - --logs '[{"category":"RequestResponse","enabled":true},{"category":"AzureOpenAIRequestUsage","enabled":true}]' -``` - -`agentops doctor` flags this automatically (rule -`waf.observability.aoai_diagnostic_categories`) and prints the exact command. - -## Required roles - -| Role | Scope | Why | -| --- | --- | --- | -| `Workbook Contributor` | Resource group that will hold the workbook | Deploy / update the `Microsoft.Insights/workbooks` resource | -| `Log Analytics Reader` | The Log Analytics workspace | Let the workbook query Azure OpenAI metrics/logs and Foundry-owned Application Insights events/spans | - -## How to use it - -Ship it with the CLI: - -```bash -# Emit the ARM template without touching Azure. -agentops telemetry dashboard deploy --dry-run - -# Deploy the workbook (needs Workbook Contributor on the resource group). -agentops telemetry dashboard deploy - -# Open the workbook in the Azure portal. -agentops telemetry dashboard open - -# Copy the workbook JSON to a local path for manual import. -agentops telemetry dashboard export --out ./foundry-ops.workbook.json -``` - -Or import it manually: **Azure portal → Monitor → Workbooks → New → Advanced -Editor**, paste the contents of `foundry-ops.workbook.json`, then Apply and Save, -selecting the target workspace and Azure OpenAI resource. - -## Authoring note (validate before GA) - -This workbook JSON was authored from the issue's KQL and the Azure Monitor -workbook schema **without a live Azure environment**. The Agent behavior -normalizer is validated with repository fixtures for both recognized table -shapes, missing optional fields, nonnumeric scores, multiple evaluators, absent -versions, and an unrecognized future schema. No live Foundry evaluation results -were invented or claimed. These synthetic fixtures validate the adapter; they -are not evidence that an automated Foundry trace-evaluation producer emits the -same event schema. - -Before relying on the workbook in production, smoke-validate it against a real -Log Analytics workspace that receives `RequestResponse`, -`AzureOpenAIRequestUsage`, Foundry `invoke_agent` spans, and -`gen_ai.evaluation.result` events. Confirm the parameters resolve, each tab -renders, data status is accurate, trace IDs correlate in Foundry Tracing, and -the derived metrics match expectations. Validate human-annotation and automated -trace-evaluation producers separately. Update the versioned normalizer instead -of silently accepting renamed properties. diff --git a/src/agentops/templates/workbooks/foundry-ops.workbook.json b/src/agentops/templates/workbooks/foundry-ops.workbook.json deleted file mode 100644 index 1af661a..0000000 --- a/src/agentops/templates/workbooks/foundry-ops.workbook.json +++ /dev/null @@ -1,683 +0,0 @@ -{ - "version": "Notebook/1.0", - "items": [ - { - "type": 1, - "content": { - "json": "# Foundry operations dashboard\nPTU capacity, traffic and tokens, latency, errors, and read-only agent behavior for Azure AI Foundry / Azure OpenAI deployments." - }, - "name": "title" - }, - { - "type": 9, - "content": { - "version": "KqlParameterItem/1.0", - "parameters": [ - { - "id": "p-subscription", - "version": "KqlParameterItem/1.0", - "name": "Subscription", - "label": "Subscription", - "type": 6, - "isRequired": true, - "typeSettings": { - "additionalResourceOptions": [], - "includeAll": false - } - }, - { - "id": "p-workspace", - "version": "KqlParameterItem/1.0", - "name": "Workspace", - "label": "Log Analytics workspace", - "type": 5, - "isRequired": true, - "resourceType": "microsoft.operationalinsights/workspaces", - "typeSettings": { - "resourceTypeFilter": { - "microsoft.operationalinsights/workspaces": true - }, - "additionalResourceOptions": [] - } - }, - { - "id": "p-resource", - "version": "KqlParameterItem/1.0", - "name": "Resource", - "label": "Azure OpenAI resource", - "type": 5, - "isRequired": true, - "resourceType": "microsoft.cognitiveservices/accounts", - "typeSettings": { - "resourceTypeFilter": { - "microsoft.cognitiveservices/accounts": true - }, - "additionalResourceOptions": [] - } - }, - { - "id": "p-timerange", - "version": "KqlParameterItem/1.0", - "name": "TimeRange", - "label": "Time range", - "type": 4, - "isRequired": true, - "typeSettings": { - "selectableValues": [ - { - "durationMs": 3600000 - }, - { - "durationMs": 14400000 - }, - { - "durationMs": 43200000 - }, - { - "durationMs": 86400000 - }, - { - "durationMs": 604800000 - }, - { - "durationMs": 2592000000 - } - ] - }, - "value": { - "durationMs": 86400000 - } - }, - { - "id": "p-deployment", - "version": "KqlParameterItem/1.0", - "name": "Deployment", - "label": "Deployment", - "type": 2, - "isRequired": false, - "multiSelect": false, - "typeSettings": { - "additionalResourceOptions": [ - "value::all" - ], - "showDefault": false - }, - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName)\n| where isnotempty(Deployment)\n| distinct Deployment\n| order by Deployment asc", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "value": "value::all" - }, - { - "id": "p-model", - "version": "KqlParameterItem/1.0", - "name": "Model", - "label": "Model", - "type": 2, - "isRequired": false, - "multiSelect": false, - "typeSettings": { - "additionalResourceOptions": [ - "value::all" - ], - "showDefault": false - }, - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Model = tostring(props.modelName)\n| where isnotempty(Model)\n| distinct Model\n| order by Model asc", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "value": "value::all" - }, - { - "id": "p-streamtype", - "version": "KqlParameterItem/1.0", - "name": "StreamType", - "label": "Streaming", - "type": 2, - "isRequired": false, - "multiSelect": false, - "typeSettings": { - "additionalResourceOptions": [] - }, - "jsonData": "[{\"value\": \"*\", \"label\": \"All\"}, {\"value\": \"Streaming\", \"label\": \"Streaming\"}, {\"value\": \"NonStreaming\", \"label\": \"Non-streaming\"}]", - "value": "*" - } - ], - "style": "pills", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces" - }, - "name": "parameters - top" - }, - { - "type": 11, - "content": { - "version": "LinkItem/1.0", - "style": "tabs", - "links": [ - { - "id": "tab-capacity", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Capacity", - "subTarget": "capacity", - "style": "link" - }, - { - "id": "tab-traffic", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Traffic and tokens", - "subTarget": "traffic", - "style": "link" - }, - { - "id": "tab-latency", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Latency", - "subTarget": "latency", - "style": "link" - }, - { - "id": "tab-errors", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Errors and throttling", - "subTarget": "errors", - "style": "link" - }, - { - "id": "tab-agent-behavior", - "cellValue": "SelectedTab", - "linkTarget": "parameter", - "linkLabel": "Agent behavior", - "subTarget": "agent-behavior", - "style": "link" - } - ] - }, - "name": "tabs" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "capacity" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "## Foundry / Azure OpenAI operations dashboard\nThis workbook is scoped **per Azure OpenAI resource** and **per Log Analytics workspace**. Pick the subscription, workspace, and Azure OpenAI resource above, then use the deployment, model, time-range, and streaming filters to narrow each tab.\n\n> **PTU_Normalizado** is a display-only scaling series (`PTU% * 100000`). It exists so the 0-100 PTU utilization percentage can share a Y axis with raw token counts in a single timechart. It is not a real token volume - read the true utilization from **PTU_Avg_Pct** / **PTU_Max_Pct**." - }, - "name": "capacity-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureMetrics\n| where MetricName in ('AzureOpenAIProvisionedManagedUtilizationV2','Ratelimit','TokenTransaction','ProcessedPromptTokens')\n| extend Deployment = tostring(split(Resource, '/')[-1])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| summarize PTU_Avg_Pct = avgif(Average, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n PTU_Max_Pct = maxif(Maximum, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n Ratelimit = avgif(Average, MetricName == 'Ratelimit'),\n ProcessedTokens = sumif(Total, MetricName in ('TokenTransaction','ProcessedPromptTokens'))\n by TimeGenerated = bin(TimeGenerated, 1m), Deployment\n| extend Spillover = max_of(0.0, ProcessedTokens - Ratelimit)\n| extend Aggregated_PTU_With_Spillover = ProcessedTokens\n| extend PTU_Normalizado = PTU_Avg_Pct * 100000\n| project TimeGenerated, PTU_Avg_Pct, PTU_Max_Pct, Ratelimit, Spillover, Aggregated_PTU_With_Spillover, PTU_Normalizado\n| order by TimeGenerated asc", - "size": 0, - "title": "PTU utilization and PAYG spillover", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "capacity-ptu" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureMetrics\n| where MetricName in ('AzureOpenAIProvisionedManagedUtilizationV2','Ratelimit','TokenTransaction','ProcessedPromptTokens')\n| extend Deployment = tostring(split(Resource, '/')[-1])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| summarize PTU_Avg_Pct = avgif(Average, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n PTU_Max_Pct = maxif(Maximum, MetricName == 'AzureOpenAIProvisionedManagedUtilizationV2'),\n Ratelimit = avgif(Average, MetricName == 'Ratelimit'),\n ProcessedTokens = sumif(Total, MetricName in ('TokenTransaction','ProcessedPromptTokens'))\n by TimeGenerated = bin(TimeGenerated, 1m), Deployment\n| extend Spillover = max_of(0.0, ProcessedTokens - Ratelimit)\n| extend Aggregated_PTU_With_Spillover = ProcessedTokens\n| extend PTU_Normalizado = PTU_Avg_Pct * 100000\n| project TimeGenerated, PTU_Avg_Pct, PTU_Max_Pct, Ratelimit, Spillover, Aggregated_PTU_With_Spillover, PTU_Normalizado\n| order by TimeGenerated asc", - "size": 0, - "title": "Capacity summary (per bin)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "capacity-ptu-table" - } - ] - }, - "name": "group-capacity" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "traffic" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "### Traffic and tokens\nCall volume and token consumption broken down by deployment and model. The PTU vs PAYG pivot splits token volume by capacity type (streaming traffic maps to PTU, non-streaming overflow to PAYG spillover)." - }, - "name": "traffic-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), Model = tostring(props.modelName), StreamType = tostring(props.streamType),\n PromptTokens = toreal(props.promptTokens[0]), GeneratedTokens = toreal(props.generatedTokens[0])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{Model}' == 'value::all' or Model == '{Model}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize AzureOpenAIRequests = count(), InputTokens = sum(PromptTokens), OutputTokens = sum(GeneratedTokens),\n ProcessedPromptTokens = sum(PromptTokens) by TimeGenerated = bin(TimeGenerated, 5m), Deployment, Model\n| extend TotalTokens = InputTokens + OutputTokens, GeneratedTokens = OutputTokens, TotalCalls = AzureOpenAIRequests\n| extend TokensPorRequest = iff(AzureOpenAIRequests > 0, ProcessedPromptTokens / AzureOpenAIRequests, 0.0)\n| order by TimeGenerated asc", - "size": 0, - "title": "Calls and tokens over time", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "traffic-tokens" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), Model = tostring(props.modelName), StreamType = tostring(props.streamType),\n PromptTokens = toreal(props.promptTokens[0]), GeneratedTokens = toreal(props.generatedTokens[0])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{Model}' == 'value::all' or Model == '{Model}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize AzureOpenAIRequests = count(), InputTokens = sum(PromptTokens), OutputTokens = sum(GeneratedTokens),\n ProcessedPromptTokens = sum(PromptTokens) by TimeGenerated = bin(TimeGenerated, 5m), Deployment, Model\n| extend TotalTokens = InputTokens + OutputTokens, GeneratedTokens = OutputTokens, TotalCalls = AzureOpenAIRequests\n| extend TokensPorRequest = iff(AzureOpenAIRequests > 0, ProcessedPromptTokens / AzureOpenAIRequests, 0.0)\n| order by TimeGenerated asc", - "size": 0, - "title": "Tokens per request (by deployment and model)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "traffic-tokens-table" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where Category == 'AzureOpenAIRequestUsage'\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType),\n PromptTokens = toreal(props.promptTokens[0]), GeneratedTokens = toreal(props.generatedTokens[0])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| extend CapacityType = iff(StreamType == 'Streaming', 'PTU', 'PAYGO_SPILLOVER')\n| summarize Tokens = sum(PromptTokens + GeneratedTokens) by TimeGenerated = bin(TimeGenerated, 5m), CapacityType\n| evaluate pivot(CapacityType, sum(Tokens))\n| order by TimeGenerated asc", - "size": 0, - "title": "PTU vs PAYG spillover (token volume)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "traffic-pivot" - } - ] - }, - "name": "group-traffic" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "latency" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "### Latency\nNormalized Azure OpenAI latency metrics (TTFT, TBT, TTLT, tokens/sec, time-to-response) plus P95/P99/Avg/Max computed from per-request `DurationMs`." - }, - "name": "latency-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureMetrics\n| where MetricName in ('AzureOpenAINormalizedTTFTInMS','AzureOpenAINormalizedTBTInMS','AzureOpenAITTLTInMS','AzureOpenAITokenPerSecond','AzureOpenAITimeToResponse','Latency')\n| extend Deployment = tostring(split(Resource, '/')[-1])\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| summarize TTFT_Ms = avgif(Average, MetricName == 'AzureOpenAINormalizedTTFTInMS'),\n TBT_Ms = avgif(Average, MetricName == 'AzureOpenAINormalizedTBTInMS'),\n TTLT_Ms = avgif(Average, MetricName == 'AzureOpenAITTLTInMS'),\n TokensPerSecond = avgif(Average, MetricName == 'AzureOpenAITokenPerSecond'),\n TimeToResponse_Ms = avgif(Average, MetricName == 'AzureOpenAITimeToResponse'),\n Latency_Ms = avgif(Average, MetricName == 'Latency') by TimeGenerated = bin(TimeGenerated, 5m)\n| order by TimeGenerated asc", - "size": 0, - "title": "Normalized latency and throughput", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "latency-platform" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| where isnotnull(DurationMs)\n| summarize P95LatencyMs = percentile(DurationMs, 95), P99LatencyMs = percentile(DurationMs, 99),\n AvgLatencyMs = avg(DurationMs), MaxLatencyMs = max(DurationMs) by TimeGenerated = bin(TimeGenerated, 5m)\n| order by TimeGenerated asc", - "size": 0, - "title": "DurationMs percentiles (P95 / P99 / Avg / Max)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "latency-percentiles" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| where isnotnull(DurationMs)\n| summarize P95LatencyMs = percentile(DurationMs, 95), P99LatencyMs = percentile(DurationMs, 99),\n AvgLatencyMs = avg(DurationMs), MaxLatencyMs = max(DurationMs) by TimeGenerated = bin(TimeGenerated, 5m)\n| order by TimeGenerated asc", - "size": 0, - "title": "Latency percentiles (per bin)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "latency-percentiles-table" - } - ] - }, - "name": "group-latency" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "errors" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "### Errors and throttling\nHTTP status breakdown (429 / 400 / 500), blocked calls, and derived throttling / error rates." - }, - "name": "errors-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType), Status = toint(ResultSignature)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize TotalRequests = count(), HTTP429 = countif(Status == 429), HTTP400 = countif(Status == 400),\n HTTP500 = countif(Status >= 500 and Status < 600), Successful = countif(Status == 200)\n by TimeGenerated = bin(TimeGenerated, 5m)\n| extend BlockedCalls = HTTP429\n| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0)\n| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0)\n| project TimeGenerated, BlockedCalls, HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct\n| order by TimeGenerated asc", - "size": 0, - "title": "HTTP status counts over time", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "errors-counts" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType), Status = toint(ResultSignature)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize TotalRequests = count(), HTTP429 = countif(Status == 429), HTTP400 = countif(Status == 400),\n HTTP500 = countif(Status >= 500 and Status < 600), Successful = countif(Status == 200)\n by TimeGenerated = bin(TimeGenerated, 5m)\n| extend BlockedCalls = HTTP429\n| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0)\n| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0)\n| project TimeGenerated, BlockedCalls, HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct\n| order by TimeGenerated asc", - "size": 0, - "title": "Throttling rate and error rate (%)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "errors-rates" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "AzureDiagnostics\n| where OperationName in ('ChatCompletions_Create','RequestResponse')\n| extend props = parse_json(properties_s)\n| extend Deployment = tostring(props.modelDeploymentName), StreamType = tostring(props.streamType), Status = toint(ResultSignature)\n| where '{Deployment}' == 'value::all' or Deployment == '{Deployment}'\n| where '{StreamType}' == '*' or StreamType == '{StreamType}'\n| summarize TotalRequests = count(), HTTP429 = countif(Status == 429), HTTP400 = countif(Status == 400),\n HTTP500 = countif(Status >= 500 and Status < 600), Successful = countif(Status == 200)\n by TimeGenerated = bin(TimeGenerated, 5m)\n| extend BlockedCalls = HTTP429\n| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0)\n| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0)\n| project TimeGenerated, BlockedCalls, HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct\n| order by TimeGenerated asc", - "size": 0, - "title": "Errors and throttling (per bin)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "errors-table" - } - ] - }, - "name": "group-errors" - }, - { - "type": 12, - "conditionalVisibility": { - "parameterName": "SelectedTab", - "comparison": "isEqualTo", - "value": "agent-behavior" - }, - "content": { - "version": "NotebookGroup/1.0", - "groupType": "editable", - "items": [ - { - "type": 1, - "content": { - "json": "## Agent behavior\n> **Preview - Foundry platform-owned evaluation signals.** Official Foundry [trace annotation documentation](https://learn.microsoft.com/azure/foundry/observability/how-to/trace-annotations#log-end-user-feedback-as-trace-annotations) verifies `gen_ai.evaluation.result` events for **human trace annotations**. Automated trace-evaluation results are shown only when the target workspace proves that it exports the same compatible event shape; that producer support is validation-dependent. AgentOps reads compatible events from Azure Monitor and does not create, schedule, gate, or edit evaluations.\n\nThe status and freshness row appears before quality results. **Observed invoke_agent invocations**, **evaluated traces**, and **evaluation events** are separate counts because one trace can produce several evaluator events and this workspace may not contain a complete invocation denominator. Trace-ID correlation does not require `gen_ai.agent.id`; missing agent/version metadata remains visible as not reported.\n\n**State guide:** `Schema unavailable` means matching events exist but the bounded `agent_behavior/v1` property mapping is not recognized. `No access` appears as the native workbook query permission error; request **Log Analytics Reader** on the workspace. `No data` means no compatible events exist in the selected time range; it can also mean automated trace evaluation does not export this event shape in the target environment. `Filter empty` means events exist but none match these filters. `Possible ingestion delay` means the newest matching event is more than 15 minutes old; it can also mean no recent compatible evaluation signal was produced." - }, - "name": "agent-behavior-note" - }, - { - "type": 9, - "content": { - "version": "KqlParameterItem/1.0", - "parameters": [ - { - "id": "p-agent-environment", - "version": "KqlParameterItem/1.0", - "name": "AgentEnvironment", - "label": "Environment (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - }, - { - "id": "p-agent-name", - "version": "KqlParameterItem/1.0", - "name": "AgentName", - "label": "Agent (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - }, - { - "id": "p-agent-version", - "version": "KqlParameterItem/1.0", - "name": "AgentVersion", - "label": "Version (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - }, - { - "id": "p-evaluator", - "version": "KqlParameterItem/1.0", - "name": "Evaluator", - "label": "Evaluator (exact; * = all)", - "type": 1, - "isRequired": true, - "value": "*" - } - ], - "style": "pills", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces" - }, - "name": "agent-behavior-filters" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nlet SchemaVersion = 'agent_behavior/v1';\nlet EvaluationEventsV1 = materialize(union isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, SourceTable='AppEvents', MonitorTraceId=tostring(column_ifexists('OperationId','')), RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, SourceTable='customEvents', MonitorTraceId=tostring(column_ifexists('operation_Id','')), RawProperties=todynamic(customDimensions))\n| extend EvaluatorRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), LabelRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), AgentRaw=tostring(RawProperties['gen_ai.agent.name']), VersionRaw=tostring(RawProperties['gen_ai.agent.version']), EnvironmentRaw=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment'])), PropertyTraceId=coalesce(tostring(RawProperties['trace_id']),tostring(RawProperties['gen_ai.trace.id']))\n| extend Agent=coalesce(AgentRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(VersionRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Environment=EnvironmentRaw, TraceId=coalesce(PropertyTraceId,MonitorTraceId), SchemaRecognized=isnotempty(EvaluatorRaw) or isnotempty(ScoreText) or isnotempty(LabelRaw)\n| extend Evaluator=iff(isempty(EvaluatorRaw),'Evaluator not reported',EvaluatorRaw), Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), TraceId=iff(isempty(TraceId),'Trace ID not reported',TraceId)\n| project EventTime,SourceTable,TraceId,Agent,Version,Environment,Evaluator,SchemaRecognized);\nlet InvokeAgentV1 = materialize(union isfuzzy=true\n (AppDependencies | project InvocationTime=TimeGenerated, SourceTable='AppDependencies', EventId=tostring(column_ifexists('Id','')), MonitorTraceId=tostring(column_ifexists('OperationId','')), RawProperties=todynamic(Properties)),\n (dependencies | project InvocationTime=timestamp, SourceTable='dependencies', EventId=tostring(column_ifexists('id','')), MonitorTraceId=tostring(column_ifexists('operation_Id','')), RawProperties=todynamic(customDimensions))\n| where tostring(RawProperties['gen_ai.operation.name']) == 'invoke_agent'\n| extend AgentId=tostring(RawProperties['gen_ai.agent.id']), AgentRaw=tostring(RawProperties['gen_ai.agent.name']), VersionRaw=tostring(RawProperties['gen_ai.agent.version']), EnvironmentRaw=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment'])), TraceId=coalesce(tostring(RawProperties['trace_id']),tostring(RawProperties['gen_ai.trace.id']),MonitorTraceId)\n| extend Agent=coalesce(AgentRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(VersionRaw,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Environment=EnvironmentRaw\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), InvocationKey=coalesce(TraceId,EventId,strcat(SourceTable,'|',format_datetime(InvocationTime,'o')))\n| project InvocationTime,InvocationKey,Agent,Version,Environment);\nlet FilteredEvents = EvaluationEventsV1 | where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}';\nlet FilteredInvocations = InvokeAgentV1 | where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}';\nlet AllSummary = EvaluationEventsV1 | summarize RawEvaluationEvents=count(), RecognizedEvaluationEvents=countif(SchemaRecognized), LatestAvailableEvent=max(EventTime);\nlet FilteredSummary = FilteredEvents | summarize EvaluationEvents=count(), EvaluatedTraces=dcountif(TraceId,TraceId != 'Trace ID not reported'), LatestMatchingEvent=max(EventTime);\nlet InvocationSummary = FilteredInvocations | summarize ObservedInvokeAgentInvocations=dcount(InvocationKey);\nAllSummary | extend JoinKey=1 | join kind=inner (FilteredSummary | extend JoinKey=1) on JoinKey | join kind=inner (InvocationSummary | extend JoinKey=1) on JoinKey\n| extend FreshnessMinutes=datetime_diff('minute',now(),LatestMatchingEvent)\n| extend DataStatus=case(RawEvaluationEvents == 0,'No data',RecognizedEvaluationEvents == 0,'Schema unavailable',EvaluationEvents == 0,'Filter empty',FreshnessMinutes > 15,'Possible ingestion delay','Ready')\n | project NormalizationVersion=SchemaVersion,ProducerSupport='Human annotations documented; automated trace-evaluation export validation-dependent',DataStatus,FreshnessMinutes,LatestMatchingEvent,ObservedInvokeAgentInvocations,EvaluatedTraces,EvaluationEvents,RawEvaluationEvents,RecognizedEvaluationEvents\n", - "size": 0, - "title": "Data status, freshness, and separate observed counts", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-status" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, SourceTable='AppEvents', RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, SourceTable='customEvents', RawProperties=todynamic(customDimensions))\n| extend EvaluatorRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), LabelRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))\n| extend SchemaRecognized=isnotempty(EvaluatorRaw) or isnotempty(ScoreText) or isnotempty(LabelRaw), MissingEvaluator=isempty(EvaluatorRaw), MissingScore=isempty(ScoreText), MissingLabel=isempty(LabelRaw)\n| where not(SchemaRecognized) or MissingEvaluator or MissingScore or MissingLabel\n| project EventTime,SourceTable,SchemaRecognized,MissingEvaluator,MissingScore,MissingLabel,RawProperties\n| order by EventTime desc\n| take 100", - "size": 0, - "title": "Schema diagnostics (raw properties retained)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-schema-diagnostics" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nlet EvaluationEventsV1 = materialize(union isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend EvaluatorRaw=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), Label=tolower(coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(EvaluatorRaw),'Evaluator not reported',EvaluatorRaw), Score=todouble(ScoreText)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), IsPass=Label in ('pass','passed','true','relevant','correct'), IsFail=Label in ('fail','failed','false','not_relevant','incorrect')\n| project EventTime,Agent,Version,Environment,Evaluator,Score,ScoreText,IsPass,IsFail);\nEvaluationEventsV1 | where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| summarize EvaluationEvents=count(), LabeledEvents=countif(IsPass or IsFail), PassingEvents=countif(IsPass), FailingEvents=countif(IsFail), NumericScores=countif(isnotnull(Score)), AverageScore=avg(Score) by Evaluator\n| extend PassRatePct=iff(LabeledEvents > 0,round(100.0 * PassingEvents / LabeledEvents,1),real(null))\n| project Evaluator,EvaluationEvents,LabeledEvents,PassRatePct,PassingEvents,FailingEvents,NumericScores,AverageScore\n| order by EvaluationEvents desc", - "size": 0, - "title": "Per-evaluator pass rate, volume, and raw-score summary", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-evaluator-summary" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), Label=tolower(coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), IsPass=Label in ('pass','passed','true','relevant','correct'), IsFail=Label in ('fail','failed','false','not_relevant','incorrect')\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| summarize LabeledEvents=countif(IsPass or IsFail), PassingEvents=countif(IsPass) by EventTime=bin(EventTime,1h),Evaluator\n| where LabeledEvents > 0\n| extend PassRatePct=round(100.0 * PassingEvents / LabeledEvents,1)\n| project EventTime,Evaluator,PassRatePct\n| order by EventTime asc", - "size": 0, - "title": "Pass-rate trend by evaluator (%)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "agent-behavior-pass-trend" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment)\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| summarize EvaluationEvents=count() by EventTime=bin(EventTime,1h),Evaluator\n| project EventTime,Evaluator,EvaluationEvents\n| order by EventTime asc", - "size": 0, - "title": "Evaluation-event volume trend by evaluator", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "timechart" - }, - "name": "agent-behavior-volume-trend" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment']))\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator), Score=todouble(ScoreText)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment)\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| where isnotnull(Score)\n| summarize AverageScore=avg(Score),MinimumScore=min(Score),MaximumScore=max(Score),NumericScoreEvents=count() by EventTime=bin(EventTime,1h),Evaluator\n| project EventTime,Evaluator,AverageScore,MinimumScore,MaximumScore,NumericScoreEvents\n| order by EventTime desc,Evaluator asc", - "size": 0, - "title": "Raw-score trend by evaluator (separate scales; do not compare evaluators)", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-score-trend" - }, - { - "type": 1, - "content": { - "json": "### Recent failed or lowest-scored evaluation events\nRaw numeric scores are ordered only within the event list and are **not comparable across unlike evaluators**. Copy a trace ID, open the matching project in **Microsoft Foundry > Tracing**, and search for that trace ID. A reliable Foundry project deep link cannot be constructed from the workbook workspace context alone." - }, - "name": "agent-behavior-trace-note" - }, - { - "type": 3, - "content": { - "version": "KqlItem/1.0", - "query": "set best_effort=true;\nunion isfuzzy=true\n (AppEvents | where Name == 'gen_ai.evaluation.result' | project EventTime=TimeGenerated, SourceTable='AppEvents', MonitorTraceId=tostring(column_ifexists('OperationId','')), RawProperties=todynamic(Properties)),\n (customEvents | where name == 'gen_ai.evaluation.result' | project EventTime=timestamp, SourceTable='customEvents', MonitorTraceId=tostring(column_ifexists('operation_Id','')), RawProperties=todynamic(customDimensions))\n| extend Evaluator=coalesce(tostring(RawProperties['gen_ai.evaluation.name']),tostring(RawProperties['evaluator'])), ScoreText=coalesce(tostring(RawProperties['gen_ai.evaluation.score.value']),tostring(RawProperties['gen_ai.evaluation.score']),tostring(RawProperties['score'])), Label=tolower(coalesce(tostring(RawProperties['gen_ai.evaluation.score.label']),tostring(RawProperties['gen_ai.evaluation.result']),tostring(RawProperties['label']))), Explanation=coalesce(tostring(RawProperties['gen_ai.evaluation.explanation']),tostring(RawProperties['gen_ai.evaluation.reason'])), AgentId=tostring(RawProperties['gen_ai.agent.id']), Agent=tostring(RawProperties['gen_ai.agent.name']), Version=tostring(RawProperties['gen_ai.agent.version']), Environment=coalesce(tostring(RawProperties['deployment.environment.name']),tostring(RawProperties['gen_ai.environment']),tostring(RawProperties['environment'])), TraceId=coalesce(tostring(RawProperties['trace_id']),tostring(RawProperties['gen_ai.trace.id']),MonitorTraceId), ResponseId=tostring(RawProperties['gen_ai.response.id']), ConversationId=tostring(RawProperties['gen_ai.conversation.id'])\n| extend Agent=coalesce(Agent,iff(AgentId contains ':',tostring(split(AgentId,':')[0]),AgentId)), Version=coalesce(Version,iff(AgentId contains ':',tostring(split(AgentId,':')[1]),'')), Evaluator=iff(isempty(Evaluator),'Evaluator not reported',Evaluator), Score=todouble(ScoreText)\n| extend Agent=iff(isempty(Agent),'Agent not reported',Agent), Version=iff(isempty(Version),'Version not reported',Version), Environment=iff(isempty(Environment),'Environment not reported',Environment), TraceId=iff(isempty(TraceId),'Trace ID not reported',TraceId), IsFail=Label in ('fail','failed','false','not_relevant','incorrect')\n| where '{AgentEnvironment}' == '*' or Environment == '{AgentEnvironment}' | where '{AgentName}' == '*' or Agent == '{AgentName}' | where '{AgentVersion}' == '*' or Version == '{AgentVersion}' | where '{Evaluator}' == '*' or Evaluator == '{Evaluator}'\n| extend ReviewPriority=case(IsFail,0,isnotnull(Score),1,2)\n| top 100 by ReviewPriority asc,Score asc,EventTime desc\n| project EventTime,Evaluator,Score,ScoreText,Label,Agent,Version,Environment,TraceId,ResponseId,ConversationId,Explanation,SourceTable\n", - "size": 0, - "title": "Recent failed or lowest-scored events with correlation IDs", - "timeContextFromParameter": "TimeRange", - "queryType": 0, - "resourceType": "microsoft.operationalinsights/workspaces", - "crossComponentResources": [ - "{Workspace}" - ], - "visualization": "table" - }, - "name": "agent-behavior-recent-traces" - } - ] - }, - "name": "group-agent-behavior" - } - ], - "isLocked": false, - "fallbackResourceIds": [ - "Azure Monitor" - ], - "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" -} diff --git a/src/agentops/templates/workbooks/queries/agent_behavior.kql b/src/agentops/templates/workbooks/queries/agent_behavior.kql deleted file mode 100644 index 1d0dbe0..0000000 --- a/src/agentops/templates/workbooks/queries/agent_behavior.kql +++ /dev/null @@ -1,222 +0,0 @@ -// Agent behavior normalization contract: agent_behavior/v1 -// -// Official Foundry docs verify gen_ai.evaluation.result events for human trace -// annotations. Automated trace-evaluation export through this same event shape -// is validation-dependent and must be proven in the target workspace. AgentOps -// reads compatible events and observed invoke_agent spans without creating, -// scheduling, or changing evaluations. The bounded query keeps an explicit -// time window and projects a stable v1 shape while preserving each raw bag. -let _startTime = ago(24h); -let _endTime = now(); -let _environment = "*"; -let _agent = "*"; -let _version = "*"; -let _evaluator = "*"; -let _normalizationVersion = "agent_behavior/v1"; -set best_effort=true; -let EvaluationEventsV1 = materialize( - union isfuzzy=true - ( - AppEvents - | where TimeGenerated between (_startTime .. _endTime) - | where Name == "gen_ai.evaluation.result" - | project - EventTime = TimeGenerated, - SourceTable = "AppEvents", - EventId = tostring(column_ifexists("Id", "")), - MonitorTraceId = tostring(column_ifexists("OperationId", "")), - RawProperties = todynamic(Properties) - ), - ( - customEvents - | where timestamp between (_startTime .. _endTime) - | where name == "gen_ai.evaluation.result" - | project - EventTime = timestamp, - SourceTable = "customEvents", - EventId = tostring(column_ifexists("id", "")), - MonitorTraceId = tostring(column_ifexists("operation_Id", "")), - RawProperties = todynamic(customDimensions) - ) - | extend - EvaluatorRaw = coalesce( - tostring(RawProperties["gen_ai.evaluation.name"]), - tostring(RawProperties["evaluator"]) - ), - ScoreText = coalesce( - tostring(RawProperties["gen_ai.evaluation.score.value"]), - tostring(RawProperties["gen_ai.evaluation.score"]), - tostring(RawProperties["score"]) - ), - LabelRaw = coalesce( - tostring(RawProperties["gen_ai.evaluation.score.label"]), - tostring(RawProperties["gen_ai.evaluation.result"]), - tostring(RawProperties["label"]) - ), - Explanation = coalesce( - tostring(RawProperties["gen_ai.evaluation.explanation"]), - tostring(RawProperties["gen_ai.evaluation.reason"]) - ), - AgentId = tostring(RawProperties["gen_ai.agent.id"]), - AgentRaw = tostring(RawProperties["gen_ai.agent.name"]), - VersionRaw = tostring(RawProperties["gen_ai.agent.version"]), - EnvironmentRaw = coalesce( - tostring(RawProperties["deployment.environment.name"]), - tostring(RawProperties["gen_ai.environment"]), - tostring(RawProperties["environment"]) - ), - PropertyTraceId = coalesce( - tostring(RawProperties["trace_id"]), - tostring(RawProperties["gen_ai.trace.id"]) - ), - ResponseId = tostring(RawProperties["gen_ai.response.id"]), - ConversationId = tostring(RawProperties["gen_ai.conversation.id"]) - | extend - Score = todouble(ScoreText), - Label = tolower(LabelRaw), - Agent = coalesce( - AgentRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[0]), AgentId) - ), - Version = coalesce( - VersionRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[1]), "") - ), - Environment = EnvironmentRaw, - TraceId = coalesce(PropertyTraceId, MonitorTraceId), - SchemaRecognized = isnotempty(EvaluatorRaw) - or isnotempty(ScoreText) - or isnotempty(LabelRaw) - | extend - Evaluator = iff(isempty(EvaluatorRaw), "Evaluator not reported", EvaluatorRaw), - Agent = iff(isempty(Agent), "Agent not reported", Agent), - Version = iff(isempty(Version), "Version not reported", Version), - Environment = iff( - isempty(Environment), - "Environment not reported", - Environment - ), - TraceId = iff(isempty(TraceId), "Trace ID not reported", TraceId), - ScoreIsNumeric = isnotempty(ScoreText) and isnotnull(Score), - IsPass = Label in ("pass", "passed", "true", "relevant", "correct"), - IsFail = Label in ( - "fail", - "failed", - "false", - "not_relevant", - "incorrect" - ), - NormalizationVersion = _normalizationVersion - | project - EventTime, - SourceTable, - EventId, - TraceId, - ResponseId, - ConversationId, - Agent, - Version, - Environment, - Evaluator, - Score, - ScoreText, - ScoreIsNumeric, - Label, - Explanation, - IsPass, - IsFail, - SchemaRecognized, - NormalizationVersion, - RawProperties -); -let InvokeAgentInvocationsV1 = materialize( - union isfuzzy=true - ( - AppDependencies - | where TimeGenerated between (_startTime .. _endTime) - | project - InvocationTime = TimeGenerated, - SourceTable = "AppDependencies", - EventId = tostring(column_ifexists("Id", "")), - MonitorTraceId = tostring(column_ifexists("OperationId", "")), - RawProperties = todynamic(Properties) - ), - ( - dependencies - | where timestamp between (_startTime .. _endTime) - | project - InvocationTime = timestamp, - SourceTable = "dependencies", - EventId = tostring(column_ifexists("id", "")), - MonitorTraceId = tostring(column_ifexists("operation_Id", "")), - RawProperties = todynamic(customDimensions) - ) - | where tostring(RawProperties["gen_ai.operation.name"]) == "invoke_agent" - | extend - AgentId = tostring(RawProperties["gen_ai.agent.id"]), - AgentRaw = tostring(RawProperties["gen_ai.agent.name"]), - VersionRaw = tostring(RawProperties["gen_ai.agent.version"]), - EnvironmentRaw = coalesce( - tostring(RawProperties["deployment.environment.name"]), - tostring(RawProperties["gen_ai.environment"]), - tostring(RawProperties["environment"]) - ), - PropertyTraceId = coalesce( - tostring(RawProperties["trace_id"]), - tostring(RawProperties["gen_ai.trace.id"]) - ) - | extend - Agent = coalesce( - AgentRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[0]), AgentId) - ), - Version = coalesce( - VersionRaw, - iff(AgentId contains ":", tostring(split(AgentId, ":")[1]), "") - ), - Environment = EnvironmentRaw, - TraceId = coalesce(PropertyTraceId, MonitorTraceId) - | extend - Agent = iff(isempty(Agent), "Agent not reported", Agent), - Version = iff(isempty(Version), "Version not reported", Version), - Environment = iff( - isempty(Environment), - "Environment not reported", - Environment - ), - InvocationKey = coalesce( - TraceId, - EventId, - strcat(SourceTable, "|", format_datetime(InvocationTime, "o")) - ), - NormalizationVersion = _normalizationVersion - | project - InvocationTime, - InvocationKey, - TraceId, - Agent, - Version, - Environment, - SourceTable, - NormalizationVersion, - RawProperties -); -EvaluationEventsV1 -| where _environment == "*" or Environment == _environment -| where _agent == "*" or Agent == _agent -| where _version == "*" or Version == _version -| where _evaluator == "*" or Evaluator == _evaluator -| summarize - EvaluationEvents = count(), - EvaluatedTraces = dcountif(TraceId, TraceId != "Trace ID not reported"), - NumericScores = countif(ScoreIsNumeric), - PassLabels = countif(IsPass), - FailLabels = countif(IsFail), - PassRatePct = round( - 100.0 * countif(IsPass) / max_of(1, countif(IsPass or IsFail)), - 1 - ), - AverageScore = avg(Score), - LatestEvent = max(EventTime) - by Evaluator, Version, NormalizationVersion -| order by EvaluationEvents desc diff --git a/src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql b/src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql deleted file mode 100644 index 5f043fb..0000000 --- a/src/agentops/templates/workbooks/queries/capacity_ptu_spillover.kql +++ /dev/null @@ -1,52 +0,0 @@ -// Capacity: PTU utilization, rate limit, and PAYG spillover -// ----------------------------------------------------------------------------- -// Source: AzureMetrics (platform metrics emitted by the Azure OpenAI resource). -// Derived metrics: PTU_Avg_Pct, PTU_Max_Pct, Ratelimit, Spillover volume, -// Aggregated_PTU_With_Spillover, PTU_Normalizado. -// -// The foundry-ops workbook binds the following parameters into this query at -// runtime. When running the query standalone in Log Analytics, edit the `let` -// bindings below to match your environment. -// {TimeRange} -> replaced by the workbook time-range picker (KQL timespan) -// {Resource} -> Azure OpenAI resource id (used as the workbook scope) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// -// Note on PTU_Normalizado: this is a DISPLAY-ONLY series (PTU% * 100000). It -// lets the 0-100 PTU utilization percentage share a Y axis with raw token -// counts in a single timechart. Do not treat it as a real token volume. -// ----------------------------------------------------------------------------- -let _bin = 1m; -let _deployment = "*"; // workbook: {Deployment} -AzureMetrics -| where TimeGenerated {TimeRange} -| where MetricName in ( - "AzureOpenAIProvisionedManagedUtilizationV2", - "Ratelimit", - "TokenTransaction", - "ProcessedPromptTokens" - ) -| extend Deployment = tostring(split(Resource, "/")[-1]) -| where _deployment == "*" or Deployment == _deployment -| summarize - PTU_Avg_Pct = avgif(Average, MetricName == "AzureOpenAIProvisionedManagedUtilizationV2"), - PTU_Max_Pct = maxif(Maximum, MetricName == "AzureOpenAIProvisionedManagedUtilizationV2"), - Ratelimit = avgif(Average, MetricName == "Ratelimit"), - ProcessedTokens = sumif(Total, MetricName in ("TokenTransaction", "ProcessedPromptTokens")) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment -// Tokens above the deployment rate limit spill over to pay-as-you-go capacity. -| extend Spillover = max_of(0.0, ProcessedTokens - Ratelimit) -| extend PTU_Tokens = ProcessedTokens - Spillover -| extend Aggregated_PTU_With_Spillover = PTU_Tokens + Spillover -// Display-only scaling series so PTU % can share a Y axis with token counts. -| extend PTU_Normalizado = PTU_Avg_Pct * 100000 -| project - TimeGenerated, - Deployment, - PTU_Avg_Pct, - PTU_Max_Pct, - Ratelimit, - Spillover, - Aggregated_PTU_With_Spillover, - PTU_Normalizado -| order by TimeGenerated asc diff --git a/src/agentops/templates/workbooks/queries/errors_throttling.kql b/src/agentops/templates/workbooks/queries/errors_throttling.kql deleted file mode 100644 index 6099381..0000000 --- a/src/agentops/templates/workbooks/queries/errors_throttling.kql +++ /dev/null @@ -1,48 +0,0 @@ -// Errors and throttling: HTTP status breakdown, throttling rate, error rate -// ----------------------------------------------------------------------------- -// Source: AzureDiagnostics (ResultSignature holds the HTTP status) and -// AzureMetrics (BlockedCalls / TotalCalls) for platform-side throttling counts. -// Derived metrics: HTTP429, HTTP400, HTTP500, TasaThrottling_Pct, TasaError_Pct, -// BlockedCalls. -// -// Workbook parameter bindings (edit the `let` lines to run standalone): -// {TimeRange} -> workbook time-range picker (KQL timespan) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// ----------------------------------------------------------------------------- -let _bin = 5m; -let _deployment = "*"; // workbook: {Deployment} -let _stream = "*"; // workbook: {StreamType} -AzureDiagnostics -| where TimeGenerated {TimeRange} -| where OperationName in ("ChatCompletions_Create", "RequestResponse") -| extend props = parse_json(properties_s) -| extend - Deployment = tostring(props.modelDeploymentName), - StreamType = tostring(props.streamType), - Status = toint(ResultSignature) -| where _deployment == "*" or Deployment == _deployment -| where _stream == "*" or StreamType == _stream -| summarize - TotalRequests = count(), - HTTP429 = countif(Status == 429), - HTTP400 = countif(Status == 400), - HTTP500 = countif(Status >= 500 and Status < 600), - Successful = countif(Status == 200) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment -| extend BlockedCalls = HTTP429 -// Throttling rate: share of requests rejected with HTTP 429. -| extend TasaThrottling_Pct = iff(TotalRequests > 0, todouble(HTTP429) / TotalRequests * 100, 0.0) -// Error rate: share of requests that did not return HTTP 200. -| extend TasaError_Pct = iff(TotalRequests > 0, todouble(TotalRequests - Successful) / TotalRequests * 100, 0.0) -| project - TimeGenerated, - Deployment, - TotalRequests, - BlockedCalls, - HTTP429, - HTTP400, - HTTP500, - TasaThrottling_Pct, - TasaError_Pct -| order by TimeGenerated asc diff --git a/src/agentops/templates/workbooks/queries/latency_percentiles.kql b/src/agentops/templates/workbooks/queries/latency_percentiles.kql deleted file mode 100644 index 5ceb5ce..0000000 --- a/src/agentops/templates/workbooks/queries/latency_percentiles.kql +++ /dev/null @@ -1,74 +0,0 @@ -// Latency: TTFT, TBT, TTLT, tokens/sec, and DurationMs percentiles -// ----------------------------------------------------------------------------- -// Source: AzureMetrics for the normalized Azure OpenAI latency metrics, and -// AzureDiagnostics (DurationMs) for per-request percentiles. -// Derived metrics: P95LatencyMs, P99LatencyMs, AvgLatencyMs, MaxLatencyMs, plus -// the platform latency metrics surfaced side by side. -// -// Workbook parameter bindings (edit the `let` lines to run standalone): -// {TimeRange} -> workbook time-range picker (KQL timespan) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// ----------------------------------------------------------------------------- -let _bin = 5m; -let _deployment = "*"; // workbook: {Deployment} -let _stream = "*"; // workbook: {StreamType} -// Platform-normalized latency and throughput metrics. -let platform = - AzureMetrics - | where TimeGenerated {TimeRange} - | where MetricName in ( - "AzureOpenAINormalizedTTFTInMS", - "AzureOpenAINormalizedTBTInMS", - "AzureOpenAITTLTInMS", - "AzureOpenAITokenPerSecond", - "AzureOpenAITimeToResponse", - "Latency" - ) - | extend Deployment = tostring(split(Resource, "/")[-1]) - | where _deployment == "*" or Deployment == _deployment - | summarize - TTFT_Ms = avgif(Average, MetricName == "AzureOpenAINormalizedTTFTInMS"), - TBT_Ms = avgif(Average, MetricName == "AzureOpenAINormalizedTBTInMS"), - TTLT_Ms = avgif(Average, MetricName == "AzureOpenAITTLTInMS"), - TokensPerSecond = avgif(Average, MetricName == "AzureOpenAITokenPerSecond"), - TimeToResponse_Ms = avgif(Average, MetricName == "AzureOpenAITimeToResponse"), - Latency_Ms = avgif(Average, MetricName == "Latency") - by TimeGenerated = bin(TimeGenerated, _bin), Deployment; -// Per-request duration percentiles from diagnostic records. -let durations = - AzureDiagnostics - | where TimeGenerated {TimeRange} - | where OperationName in ("ChatCompletions_Create", "RequestResponse") - | extend props = parse_json(properties_s) - | extend - Deployment = tostring(props.modelDeploymentName), - StreamType = tostring(props.streamType) - | where _deployment == "*" or Deployment == _deployment - | where _stream == "*" or StreamType == _stream - | where isnotnull(DurationMs) - | summarize - P95LatencyMs = percentile(DurationMs, 95), - P99LatencyMs = percentile(DurationMs, 99), - AvgLatencyMs = avg(DurationMs), - MaxLatencyMs = max(DurationMs) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment; -platform -| join kind=fullouter durations on TimeGenerated, Deployment -| extend - TimeGenerated = coalesce(TimeGenerated, TimeGenerated1), - Deployment = coalesce(Deployment, Deployment1) -| project - TimeGenerated, - Deployment, - TTFT_Ms, - TBT_Ms, - TTLT_Ms, - TokensPerSecond, - TimeToResponse_Ms, - Latency_Ms, - P95LatencyMs, - P99LatencyMs, - AvgLatencyMs, - MaxLatencyMs -| order by TimeGenerated asc diff --git a/src/agentops/templates/workbooks/queries/traffic_tokens.kql b/src/agentops/templates/workbooks/queries/traffic_tokens.kql deleted file mode 100644 index b8946cf..0000000 --- a/src/agentops/templates/workbooks/queries/traffic_tokens.kql +++ /dev/null @@ -1,61 +0,0 @@ -// Traffic and tokens: call volume, token consumption, and PTU vs PAYG split -// ----------------------------------------------------------------------------- -// Source: AzureDiagnostics (Category == "AzureOpenAIRequestUsage") for real -// per-model token usage, joined with AzureMetrics for platform call volume. -// Derived metrics: TokensPorRequest, InputTokens, OutputTokens, TotalTokens, -// ProcessedPromptTokens, GeneratedTokens, and a PTU vs PAYG -// spillover pivot. -// -// Workbook parameter bindings (edit the `let` lines to run standalone): -// {TimeRange} -> workbook time-range picker (KQL timespan) -// {Deployment} -> modelDeploymentName filter, or "*" for all deployments -// {Model} -> modelName filter, or "*" for all models -// {StreamType} -> "Streaming", "NonStreaming", or "*" for both -// ----------------------------------------------------------------------------- -let _bin = 5m; -let _deployment = "*"; // workbook: {Deployment} -let _model = "*"; // workbook: {Model} -let _stream = "*"; // workbook: {StreamType} -AzureDiagnostics -| where TimeGenerated {TimeRange} -| where Category == "AzureOpenAIRequestUsage" -| extend props = parse_json(properties_s) -| extend - Deployment = tostring(props.modelDeploymentName), - Model = tostring(props.modelName), - ModelVersion = tostring(props.modelVersion), - StreamType = tostring(props.streamType), - PromptTokens = toreal(props.promptTokens[0]), - GeneratedTokens = toreal(props.generatedTokens[0]) -| where _deployment == "*" or Deployment == _deployment -| where _model == "*" or Model == _model -| where _stream == "*" or StreamType == _stream -| summarize - AzureOpenAIRequests = count(), - InputTokens = sum(PromptTokens), - OutputTokens = sum(GeneratedTokens), - ProcessedPromptTokens = sum(PromptTokens) - by TimeGenerated = bin(TimeGenerated, _bin), Deployment, Model, StreamType -| extend TotalTokens = InputTokens + OutputTokens -| extend GeneratedTokens = OutputTokens -| extend TotalCalls = AzureOpenAIRequests -// Average tokens billed per request over the bin. -| extend TokensPorRequest = iff(AzureOpenAIRequests > 0, ProcessedPromptTokens / AzureOpenAIRequests, 0.0) -// PTU vs PAYG split: streaming and provisioned traffic run on PTU capacity; -// non-streaming overflow is treated as the PAYG spillover pivot column. -| extend CapacityType = iff(StreamType == "Streaming", "PTU", "PAYGO_SPILLOVER") -| project - TimeGenerated, - Deployment, - Model, - StreamType, - CapacityType, - TotalCalls, - AzureOpenAIRequests, - InputTokens, - OutputTokens, - TotalTokens, - ProcessedPromptTokens, - GeneratedTokens, - TokensPorRequest -| order by TimeGenerated asc diff --git a/src/agentops/utils/foundry_discovery.py b/src/agentops/utils/foundry_discovery.py index 8737b3a..50fb160 100644 --- a/src/agentops/utils/foundry_discovery.py +++ b/src/agentops/utils/foundry_discovery.py @@ -283,6 +283,81 @@ def resolve_appinsights_connection(project_endpoint: str) -> Optional[str]: return conn +def resolve_appinsights_resource_id_with_reason( + project_endpoint: str, +) -> Tuple[Optional[str], Optional[str]]: + """Return the ARM resource ID of the Foundry-linked App Insights resource. + + Connection metadata is credential-free, so this works for both API-key and + ProjectManagedIdentity connections without requesting connection secrets. + """ + if not project_endpoint: + return None, "no AZURE_AI_FOUNDRY_PROJECT_ENDPOINT set" + + cache_key = f"appinsights-resource:{project_endpoint}" + cached = _lookup(cache_key) + if cached is not None: + return cached + + try: + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + except ImportError: + reason = ( + "azure-ai-projects / azure-identity not installed in the cockpit's " + "Python environment. Install with " + "`pip install azure-ai-projects azure-identity`." + ) + _store(cache_key, None, reason) + return None, reason + + try: + credential = DefaultAzureCredential( + exclude_developer_cli_credential=True, + process_timeout=30, + ) + client = AIProjectClient(endpoint=project_endpoint, credential=credential) + connections = getattr(client, "connections", None) + list_connections = getattr(connections, "list", None) + if not callable(list_connections): + reason = ( + "AIProjectClient has no connections.list helper " + "(azure-ai-projects too old)." + ) + _store(cache_key, None, reason) + return None, reason + + for connection in list_connections(): + connection_type = str(getattr(connection, "type", "") or "").lower() + target = str(getattr(connection, "target", "") or "").strip() + is_app_insights = ( + "application_insights" in connection_type + or "applicationinsights" in connection_type + ) + is_resource_id = ( + target.lower().startswith("/subscriptions/") + and "/providers/microsoft.insights/components/" in target.lower() + ) + if is_app_insights and is_resource_id: + _store(cache_key, target, None) + return target, None + except Exception as exc: # noqa: BLE001 + reason = _summarize_discovery_exception( + exc, + context="Foundry App Insights connection metadata discovery", + ) + _store(cache_key, None, reason) + return None, reason + + reason = ( + "Foundry returned no Application Insights connection metadata. Wire " + "one in: Project details \u2192 Connected resources \u2192 " + "Add connection \u2192 Application Insights." + ) + _store(cache_key, None, reason) + return None, reason + + def resolve_appinsights_connection_from_env() -> Optional[str]: """Resolve using ``AZURE_AI_FOUNDRY_PROJECT_ENDPOINT`` if set.""" endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") @@ -300,3 +375,13 @@ def resolve_appinsights_connection_from_env_with_reason() -> Tuple[ if not endpoint: return None, "no AZURE_AI_FOUNDRY_PROJECT_ENDPOINT set" return resolve_appinsights_connection_with_reason(endpoint) + + +def resolve_appinsights_resource_id_from_env_with_reason() -> Tuple[ + Optional[str], Optional[str] +]: + """Resolve App Insights ARM metadata from the configured Foundry project.""" + endpoint = os.getenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT") + if not endpoint: + return None, "no AZURE_AI_FOUNDRY_PROJECT_ENDPOINT set" + return resolve_appinsights_resource_id_with_reason(endpoint) diff --git a/tests/unit/test_agent_identity_service.py b/tests/unit/test_agent_identity_service.py index 147c04d..c4e4940 100644 --- a/tests/unit/test_agent_identity_service.py +++ b/tests/unit/test_agent_identity_service.py @@ -249,6 +249,16 @@ def test_resolve_display_name_falls_back_to_existing_record(tmp_path: Path) -> N assert resolve_display_name(tmp_path) == "from-record" +def test_resolve_display_name_from_hosted_agent_url(tmp_path: Path) -> None: + _write_config( + tmp_path, + "agent: https://example.services.ai.azure.com/api/projects/demo/" + "agents/helpdeskbot/versions/13\n", + ) + + assert resolve_display_name(tmp_path) == "helpdeskbot" + + def test_resolve_registration_inputs_returns_name_and_sponsor(tmp_path: Path) -> None: _write_config( tmp_path, diff --git a/tests/unit/test_agent_posture_rules.py b/tests/unit/test_agent_posture_rules.py index fc4698d..55849e0 100644 --- a/tests/unit/test_agent_posture_rules.py +++ b/tests/unit/test_agent_posture_rules.py @@ -10,9 +10,6 @@ from agentops.agent.checks.posture_rules.diagnostics import ( evaluate as diagnostics_rule, ) -from agentops.agent.checks.posture_rules.aoai_diagnostic_categories import ( - evaluate as aoai_diag_rule, -) from agentops.agent.checks.posture_rules.local_auth import ( evaluate as local_auth_rule, ) @@ -313,71 +310,4 @@ def test_rule_registry_only_contains_complementary_rules() -> None: "waf.security.local_auth_disabled", "waf.security.managed_identity", "waf.security.diagnostic_settings", - "waf.observability.aoai_diagnostic_categories", } - - -# --------------------------------------------------------------------------- -# aoai_diagnostic_categories (WAF-AI Operational Excellence) -# --------------------------------------------------------------------------- - - -def test_aoai_diag_rule_passes_when_both_categories_enabled() -> None: - payload = _payload( - diagnostic_settings=[ - DiagnosticSettingSnapshot( - name="default", - workspace_id="/subscriptions/s/workspaces/log", - enabled_log_categories=[ - "RequestResponse", - "AzureOpenAIRequestUsage", - ], - ) - ] - ) - assert aoai_diag_rule(payload, "azure_resources") == [] - - -def test_aoai_diag_rule_fires_when_usage_category_missing() -> None: - payload = _payload( - diagnostic_settings=[ - DiagnosticSettingSnapshot( - name="default", - workspace_id="/subscriptions/s/workspaces/log", - enabled_log_categories=["RequestResponse"], - ) - ] - ) - findings = aoai_diag_rule(payload, "azure_resources") - assert len(findings) == 1 - finding = findings[0] - assert finding.id == "waf.observability.aoai_diagnostic_categories" - assert finding.severity is Severity.WARNING - assert finding.category is Category.OPERATIONAL_EXCELLENCE - assert finding.evidence["missing_categories"] == ["AzureOpenAIRequestUsage"] - # The recommendation must carry the exact az fix command. - assert "az monitor diagnostic-settings create" in finding.recommendation - assert "AzureOpenAIRequestUsage" in finding.recommendation - - -def test_aoai_diag_rule_fires_when_no_categories() -> None: - payload = _payload( - diagnostic_settings=[ - DiagnosticSettingSnapshot( - name="default", - workspace_id="/subscriptions/s/workspaces/log", - enabled_log_categories=[], - ) - ] - ) - findings = aoai_diag_rule(payload, "azure_resources") - assert len(findings) == 1 - assert findings[0].evidence["missing_categories"] == [ - "RequestResponse", - "AzureOpenAIRequestUsage", - ] - - -def test_aoai_diag_rule_noop_without_account() -> None: - payload = AzureResourcesPayload(account=None) - assert aoai_diag_rule(payload, "azure_resources") == [] diff --git a/tests/unit/test_agentops_config.py b/tests/unit/test_agentops_config.py index 4e0b30d..16620b3 100644 --- a/tests/unit/test_agentops_config.py +++ b/tests/unit/test_agentops_config.py @@ -304,110 +304,6 @@ def test_minimal_config(self, tmp_path) -> None: assert cfg.agent == "my-rag:3" assert cfg.thresholds == {} assert cfg.response_source == "agent" - assert cfg.telemetry_imports == [] - - def test_accepts_telemetry_import_config(self) -> None: - cfg = AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "response_source": "dataset", - "telemetry_imports": [ - { - "name": "prod", - "source": "azure-monitor", - "target": "application-insights", - "resource_id": "$APPINSIGHTS_RESOURCE_ID", - "time_range": {"lookback_days": 14}, - "filters": {"customDimensions.agent": "support"}, - "fields": { - "input": "customDimensions.question", - "response": "customDimensions.answer", - }, - "privacy": {"redact_fields": ["token"], "max_field_length": 500}, - "output": { - "path": ".agentops/data/prod.jsonl", - "label_mode": "pending", - }, - } - ], - } - ) - - item = cfg.telemetry_imports[0] - assert cfg.response_source == "dataset" - assert item.name == "prod" - assert item.source == "azure-monitor" - assert item.target == "application-insights" - assert item.resource_id == "$APPINSIGHTS_RESOURCE_ID" - assert item.time_range.lookback_days == 14 - assert item.output.label_mode == "pending" - - def test_telemetry_import_rejects_unknown_fields(self) -> None: - with pytest.raises(ValidationError): - AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "log-analytics", - "workspace_id": "workspace", - "surprise": True, - } - ], - } - ) - - def test_telemetry_import_time_range_requires_one_mode(self) -> None: - with pytest.raises(ValidationError, match="cannot mix"): - AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "log-analytics", - "workspace_id": "workspace", - "time_range": { - "from": "2026-06-01T00:00:00Z", - "to": "2026-06-02T00:00:00Z", - "lookback_days": 7, - }, - } - ], - } - ) - - def test_telemetry_import_accepts_explicit_time_range(self) -> None: - cfg = AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "my-rag:3", - "dataset": "./qa.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "log-analytics", - "workspace_id": "workspace", - "time_range": { - "from": "2026-06-01T00:00:00Z", - "to": "2026-06-02T00:00:00Z", - }, - } - ], - } - ) - - time_range = cfg.telemetry_imports[0].time_range - assert time_range.from_ == "2026-06-01T00:00:00Z" - assert time_range.to == "2026-06-02T00:00:00Z" - assert time_range.lookback_days is None def test_resolved_target(self) -> None: cfg = AgentOpsConfig(version=1, agent="my-rag:3", dataset="./qa.jsonl") diff --git a/tests/unit/test_cli_commands.py b/tests/unit/test_cli_commands.py index 3bb6feb..35b024d 100644 --- a/tests/unit/test_cli_commands.py +++ b/tests/unit/test_cli_commands.py @@ -1,9 +1,6 @@ from typer.testing import CliRunner from agentops.cli.app import app -from agentops.services.telemetry_import import TelemetryImportPreview - - runner = CliRunner() @@ -49,11 +46,11 @@ def test_eval_help_does_not_expose_compare_subcommand() -> None: assert "compare" not in stripped -def test_planned_command_groups_removed() -> None: - """Stub command groups (monitor/model/dataset/config) are gone in 1.0. +def test_removed_command_groups_are_not_wired() -> None: + """Retired and former stub command groups are absent. `cockpit` is now the real command that opens the local UI.""" - for group in ("monitor", "model", "dataset", "config"): + for group in ("monitor", "model", "dataset", "config", "telemetry"): result = runner.invoke(app, [group, "--help"]) assert result.exit_code != 0, f"unexpected: 'agentops {group}' is still wired" @@ -84,102 +81,3 @@ def test_agent_command_group_wired() -> None: stripped = _strip_ansi(result.stdout) assert "analyze" in stripped assert "serve" in stripped - - -def test_telemetry_validate_uses_named_import(tmp_path, monkeypatch) -> None: - config = tmp_path / "agentops.yaml" - config.write_text( - "\n".join( - [ - "version: 1", - "agent: support-agent:1", - "dataset: .agentops/data/smoke.jsonl", - "telemetry_imports:", - " - name: prod", - " target: log-analytics", - " workspace_id: workspace", - ] - ), - encoding="utf-8", - ) - monkeypatch.setattr( - "agentops.services.telemetry_import.validate_telemetry_import", - lambda _item: [], - ) - - result = runner.invoke(app, ["telemetry", "validate", "prod", "--config", str(config)]) - - assert result.exit_code == 0, result.output - assert "prod" in result.output - assert "valid" in result.output - - -def test_telemetry_preview_prints_service_preview(tmp_path, monkeypatch) -> None: - config = tmp_path / "agentops.yaml" - config.write_text( - "version: 1\n" - "agent: support-agent:1\n" - "dataset: .agentops/data/smoke.jsonl\n" - "telemetry_imports:\n" - " - name: prod\n" - " target: log-analytics\n" - " workspace_id: workspace\n", - encoding="utf-8", - ) - - def fake_preview(item, *, rows=None, apply=False): - return TelemetryImportPreview( - config=item, - output_path=tmp_path / "prod.jsonl", - manifest_path=tmp_path / "prod-manifest.json", - rows=[{"input": "hello", "response": "world"}], - ) - - monkeypatch.setattr( - "agentops.services.telemetry_import.preview_telemetry_import", - fake_preview, - ) - - result = runner.invoke( - app, - ["telemetry", "preview", "prod", "--rows", "1", "--config", str(config)], - ) - - assert result.exit_code == 0, result.output - assert "AgentOps telemetry import" in result.output - assert "hello" in result.output - - -def test_telemetry_import_requires_apply_to_write(tmp_path, monkeypatch) -> None: - config = tmp_path / "agentops.yaml" - config.write_text( - "version: 1\n" - "agent: support-agent:1\n" - "dataset: .agentops/data/smoke.jsonl\n" - "telemetry_imports:\n" - " - name: prod\n" - " target: log-analytics\n" - " workspace_id: workspace\n", - encoding="utf-8", - ) - calls = [] - - def fake_preview(item, *, rows=None, apply=False): - calls.append(apply) - return TelemetryImportPreview( - config=item, - output_path=tmp_path / "prod.jsonl", - manifest_path=tmp_path / "prod-manifest.json", - rows=[], - ) - - monkeypatch.setattr( - "agentops.services.telemetry_import.preview_telemetry_import", - fake_preview, - ) - - result = runner.invoke(app, ["telemetry", "import", "prod", "--config", str(config)]) - - assert result.exit_code == 0, result.output - assert calls == [False] - assert "Dry run only" in result.output diff --git a/tests/unit/test_cli_dashboard.py b/tests/unit/test_cli_dashboard.py deleted file mode 100644 index c064ac2..0000000 --- a/tests/unit/test_cli_dashboard.py +++ /dev/null @@ -1,156 +0,0 @@ -"""CLI tests for the ``agentops telemetry dashboard`` sub-app. - -The Azure logic lives in :mod:`agentops.services.dashboard`; these tests keep -Azure out of the loop by exercising ``deploy --dry-run`` (which never touches -Azure), ``open --print-url`` and ``export`` (both pure), plus the ``--help`` -and ``explain`` surfaces. -""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -from typer.testing import CliRunner - -from agentops.cli.app import app - - -runner = CliRunner() - - -def _strip_ansi(text: str) -> str: - return re.sub(r"\x1b\[[0-9;]*m", "", text) - - -# --------------------------------------------------------------------------- -# help / discoverability -# --------------------------------------------------------------------------- -def test_dashboard_help_lists_commands() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "--help"]) - assert result.exit_code == 0 - stripped = _strip_ansi(result.stdout) - assert "deploy" in stripped - assert "open" in stripped - assert "export" in stripped - assert "agent behavior" in stripped.lower() - - -def test_dashboard_deploy_help() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "deploy", "--help"]) - assert result.exit_code == 0 - stripped = _strip_ansi(result.stdout) - assert "--dry-run" in stripped - assert "--subscription" in stripped - assert "--resource-group" in stripped - assert "--workspace-id" in stripped - assert "--name" in stripped - - -def test_dashboard_open_help() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "open", "--help"]) - assert result.exit_code == 0 - assert "--print-url" in _strip_ansi(result.stdout) - - -def test_dashboard_export_help() -> None: - result = runner.invoke(app, ["telemetry", "dashboard", "export", "--help"]) - assert result.exit_code == 0 - assert "--out" in _strip_ansi(result.stdout) - - -# --------------------------------------------------------------------------- -# explain pages -# --------------------------------------------------------------------------- -def test_dashboard_explain_pages_exit_zero() -> None: - for path in ( - ["telemetry", "dashboard", "explain"], - ["telemetry", "dashboard", "deploy", "explain"], - ["telemetry", "dashboard", "open", "explain"], - ["telemetry", "dashboard", "export", "explain"], - ): - result = runner.invoke(app, path) - assert result.exit_code == 0, f"{path} -> {result.stdout}" - - -# --------------------------------------------------------------------------- -# deploy --dry-run (no Azure) -# --------------------------------------------------------------------------- -def test_dashboard_deploy_dry_run_emits_arm_template(tmp_path: Path) -> None: - result = runner.invoke( - app, - [ - "telemetry", - "dashboard", - "deploy", - "--dry-run", - "--subscription", - "sub", - "--resource-group", - "rg", - "--dir", - str(tmp_path), - ], - ) - assert result.exit_code == 0, result.stdout - # The dry-run prints the ARM template to stdout and an advisory note to - # stderr (mixed into stdout by CliRunner); parse just the JSON object. - obj, _end = json.JSONDecoder().raw_decode(result.stdout[result.stdout.index("{") :]) - template = obj - assert template["resources"][0]["type"] == "Microsoft.Insights/workbooks" - assert template["resources"][0]["properties"]["serializedData"] - - -# --------------------------------------------------------------------------- -# open --print-url (no browser, no Azure) -# --------------------------------------------------------------------------- -def test_dashboard_open_print_url(tmp_path: Path) -> None: - result = runner.invoke( - app, - [ - "telemetry", - "dashboard", - "open", - "--print-url", - "--subscription", - "sub", - "--resource-group", - "rg", - "--dir", - str(tmp_path), - ], - ) - assert result.exit_code == 0, result.stdout - assert "https://portal.azure.com/" in result.stdout - - -def test_dashboard_open_falls_back_to_gallery_without_target(tmp_path: Path) -> None: - result = runner.invoke( - app, - ["telemetry", "dashboard", "open", "--print-url", "--dir", str(tmp_path)], - ) - assert result.exit_code == 0, result.stdout - # No subscription/rg discoverable -> gallery deep link. - assert "WorkbookMenuBlade" in result.stdout - - -# --------------------------------------------------------------------------- -# export -# --------------------------------------------------------------------------- -def test_dashboard_export_writes_file(tmp_path: Path) -> None: - out = tmp_path / "foundry-ops.workbook.json" - result = runner.invoke(app, ["telemetry", "dashboard", "export", "--out", str(out)]) - assert result.exit_code == 0, result.stdout - assert out.is_file() - # The exported file is valid JSON identical to the packaged template. - from agentops.services import dashboard as dash - - assert json.loads(out.read_text(encoding="utf-8")) == dash.load_workbook_content() - - -def test_dashboard_export_creates_parent_dirs(tmp_path: Path) -> None: - out = tmp_path / "nested" / "dir" / "wb.json" - result = runner.invoke(app, ["telemetry", "dashboard", "export", "--out", str(out)]) - assert result.exit_code == 0, result.stdout - assert out.is_file() diff --git a/tests/unit/test_cockpit.py b/tests/unit/test_cockpit.py index 5c3a391..e31ea59 100644 --- a/tests/unit/test_cockpit.py +++ b/tests/unit/test_cockpit.py @@ -116,88 +116,11 @@ def _write_eval_run( def test_empty_workspace_yields_empty_state(tmp_path: Path): payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - assert payload["eval"]["has_runs"] is False - assert payload["metrics"] == [] assert payload["watchdog"]["has_history"] is False + assert len(payload["readiness"]["checks"]) == 13 html = render_cockpit_html(payload) - assert "No eval runs yet" in html assert "No analysis history yet" in html - assert "agentops eval run" in html - - -def test_cockpit_loads_eval_runs(tmp_path: Path): - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-11T20-00-00Z", - passed=True, - metrics={"coherence": 4.5, "similarity": 4.0, "fluency": 3.7, "f1_score": 0.9}, - ) - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-11T21-00-00Z", - passed=False, - metrics={"coherence": 4.0, "similarity": 3.0, "fluency": 3.0, "f1_score": 0.6}, - target="agent-smoke:3", - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - assert payload["eval"]["has_runs"] is True - eval_keys = {c["key"] for c in payload["eval"]["cards"]} - assert "total_runs" in eval_keys - assert "pass_rate" in eval_keys - assert "latest_run" in eval_keys - # Latest target wins. - latest_card = next(c for c in payload["eval"]["cards"] if c["key"] == "latest_run") - assert latest_card["value"] == "agent-smoke:3" - assert latest_card["badge"]["tone"] == "crit" - - metric_keys = {c["key"] for c in payload["metrics"]} - assert {"coherence", "similarity", "fluency", "f1_score"} <= metric_keys - - -def test_pass_rate_badge_reflects_history(tmp_path: Path): - for i in range(4): - _write_eval_run( - tmp_path, - timestamp_dir=f"2026-05-11T0{i}-00-00Z", - passed=True, - metrics={"coherence": 4.0}, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - pass_card = next(c for c in payload["eval"]["cards"] if c["key"] == "pass_rate") - assert pass_card["value"] == "100%" - assert pass_card["badge"]["tone"] == "ok" - - -def test_metric_trend_badge_detects_regression_for_quality(tmp_path: Path): - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T01-00-00Z", passed=True, - metrics={"coherence": 5.0}, - ) - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T02-00-00Z", passed=True, - metrics={"coherence": 3.0}, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - coh = next(c for c in payload["metrics"] if c["key"] == "coherence") - assert coh["badge"]["label"] == "regressed" - assert coh["badge"]["tone"] == "warn" - - -def test_metric_trend_badge_treats_latency_inversely(tmp_path: Path): - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T01-00-00Z", passed=True, - metrics={"avg_latency_seconds": 5.0}, - ) - _write_eval_run( - tmp_path, timestamp_dir="2026-05-11T02-00-00Z", passed=True, - metrics={"avg_latency_seconds": 2.0}, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - lat = next(c for c in payload["metrics"] if c["key"] == "avg_latency_seconds") - # Latency dropping is an improvement, not a regression. - assert lat["badge"]["label"] == "improved" - assert lat["badge"]["tone"] == "ok" + assert "NO-GO" in html def test_telemetry_status_reflects_env(tmp_path: Path, monkeypatch): @@ -216,6 +139,44 @@ def test_telemetry_status_reflects_env(tmp_path: Path, monkeypatch): assert payload["telemetry"]["source"] == "env" +def test_telemetry_status_accepts_foundry_project_managed_identity(monkeypatch): + from agentops.agent.cockpit import _telemetry_status + from agentops.utils import foundry_discovery + + resource_id = ( + "/subscriptions/000/resourceGroups/rg/providers/" + "Microsoft.Insights/components/appi-pmi" + ) + monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) + monkeypatch.delenv("AGENTOPS_APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) + monkeypatch.delenv("AGENTOPS_OTLP_ENDPOINT", raising=False) + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://x.services.ai.azure.com/api/projects/pmi", + ) + monkeypatch.setattr( + foundry_discovery, + "resolve_appinsights_connection_from_env_with_reason", + lambda: ( + None, + "Foundry Application Insights connection uses " + "ProjectManagedIdentity; API Key credentials are not required.", + ), + ) + monkeypatch.setattr( + foundry_discovery, + "resolve_appinsights_resource_id_from_env_with_reason", + lambda: (resource_id, None), + ) + + status = _telemetry_status() + + assert status["enabled"] is True + assert status["source"] == "foundry_project_connection" + assert status["resource_id"] == resource_id + assert status["portal_url"].endswith(f"#resource{resource_id}/overview") + + def test_watchdog_section_surfaces_latest_findings(tmp_path: Path): """The watchdog section exposes the latest run's findings (sorted by severity desc) instead of per-category trend charts.""" @@ -294,7 +255,9 @@ def test_finding_recommendation_renders_safe_markdown(tmp_path: Path): assert "<script>alert(1)</script>" in html -def test_html_includes_all_sections_when_data_present(tmp_path: Path, monkeypatch): +def test_html_contains_exactly_five_sections_in_required_order( + tmp_path: Path, monkeypatch, +): _set_appinsights_env(monkeypatch) _write_eval_run( tmp_path, timestamp_dir="2026-05-11T01-00-00Z", passed=True, @@ -310,122 +273,74 @@ def test_html_includes_all_sections_when_data_present(tmp_path: Path, monkeypatc payload = build_cockpit_payload(tmp_path, time_range=_WIDE) html = render_cockpit_html(payload) - # New strategic sections. - assert "Foundry connection" in html - assert "Foundry launchpad" in html - assert "Azure Monitor" in html - assert "Observability readiness" in html - assert "Next actions" in html - # Consolidated top status cards ("can I ship?"). - assert 'id="section-status-cards"' in html - assert "Can I ship?" in html - # Gate summary sections: eval + quality gate are merged into one - # "Eval gates" section with two subgroups. - assert "Eval gates" in html - assert "Eval gate summary" in html - assert "AgentOps Doctor" in html - assert "CI/CD" in html - assert "Quality gate summary" in html - # Production telemetry is now rendered as a minimal signal into - # Foundry Monitor; the section title is "Production signal". - assert "Production signal" in html - assert "Full view in Foundry Monitor" in html - assert "Fast health snapshot from App Insights" in html - assert "Foundry Monitor for the full production view" in html - assert "Quality gate trends computed from AgentOps result artifacts" in html - assert "AgentOps gate history from local artifacts and CI runs" in html - assert "Foundry connection') status_cards_pos = html.find('id="section-status-cards"') - actions_pos = html.find('Next actions') + connections_pos = html.find('Connections') readiness_pos = html.find('Observability readiness') doctor_pos = html.find('AgentOps Doctor') - eval_gates_pos = html.find('Eval gates') - prod_pos = html.find('Production signal') - deploy_pos = html.find('CI/CD') - launchpad_pos = html.find('Foundry launchpad') - for pos in ( - connection_pos, status_cards_pos, actions_pos, readiness_pos, - doctor_pos, eval_gates_pos, prod_pos, deploy_pos, launchpad_pos, - ): - assert pos != -1, "missing strategic section in cockpit HTML" - assert ( - connection_pos < status_cards_pos < actions_pos < readiness_pos - < doctor_pos < eval_gates_pos < prod_pos < deploy_pos < launchpad_pos + actions_pos = html.find('Next actions') + assert -1 not in ( + status_cards_pos, + connections_pos, + readiness_pos, + doctor_pos, + actions_pos, ) - # Both subgroup labels live inside the merged Eval gates section, eval - # before quality. assert ( - eval_gates_pos - < html.find("Eval gate summary") - < html.find("Quality gate summary") - ) - # Detail sections collapse by default; the top status/next-actions - # sections stay open. At least one collapsed
is present. - assert '
Eval gates', + 'Production signal', + 'CI/CD Pipelines', + 'Foundry launchpad', + "range-pills", + "refreshSelect", + "Auto-refresh", + "window:", + ): + assert removed not in html - The former single-tile "Azure Monitor" group is folded into the - Foundry project subgroup (App Insights + the new Foundry operations - dashboard tile), so there is one place to look instead of a duplicated - one-tile group. - """ - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - open_panel = payload["open_in_foundry"] - groups = open_panel.get("groups") or [] - keys = [g.get("key") for g in groups] - assert keys == ["agent", "project"], ( - "Agent links must precede project links; Azure Monitor is folded " - "into the project group" +def test_connections_only_contains_foundry_and_github(tmp_path: Path, monkeypatch): + monkeypatch.setenv( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", + "https://account.services.ai.azure.com/api/projects/project", ) - - agent_titles = {t["title"] for t in groups[0]["targets"]} - assert {"Agent build", "Monitor", "Traces"}.issubset(agent_titles) - - project_titles = {t["title"] for t in groups[1]["targets"]} - assert { - "Evaluations", - "Datasets", - "Red Teaming", - "Operate overview", - "Foundry operations dashboard", - "App Insights", - }.issubset(project_titles) - - # The new Foundry operations dashboard tile renders right after - # Operate overview, and the two descriptions must not read the same. - project_keys = [t["key"] for t in groups[1]["targets"]] - assert project_keys.index("foundry_ops_dashboard") == ( - project_keys.index("operate") + 1 + monkeypatch.setattr( + "agentops.agent.cockpit._resolve_github_repository", + lambda _workspace: { + "name": "owner/repo", + "url": "https://github.com/owner/repo", + }, ) - descriptions = {t["key"]: t.get("description", "") for t in groups[1]["targets"]} - assert descriptions["operate"] != descriptions["foundry_ops_dashboard"] - + payload = build_cockpit_payload(tmp_path) + items = payload["connections"]["items"] + assert [item["title"] for item in items] == [ + "Foundry project", + "GitHub repository", + ] html = render_cockpit_html(payload) - # Subheaders render (only the two remaining groups). - assert ">Configured agent<" in html - assert ">Foundry project<" in html - # The legacy flat ``targets`` key is kept for backwards-compat and - # combines both groups in display order. - flat_keys = [t["key"] for t in open_panel["targets"]] - assert flat_keys[0] == "agent" - assert flat_keys[-1] == "app_insights" - - -def test_readiness_splits_tracing_and_includes_continuous_eval(tmp_path: Path): - """Readiness now lists separate server-side and client-side tracing - rows plus a dedicated continuous-evaluation row sourced from the - latest Doctor analysis.""" + assert "Open in Foundry" in html + assert "Open in GitHub" in html + assert "Azure tenant" not in html + assert "Application Insights
" not in html + + +def test_readiness_splits_connection_and_instrumentation(tmp_path: Path): + """Readiness separates App Insights linkage from agent instrumentation.""" from agentops.agent.cockpit import ( _build_readiness_checklist, _render_readiness_section, @@ -441,8 +356,8 @@ def test_readiness_splits_tracing_and_includes_continuous_eval(tmp_path: Path): tmp_path, telemetry, deployments, watchdog=None, ) titles = [c["title"] for c in readiness["checks"]] - assert any("Server-side tracing" in t for t in titles) - assert any("Client-side tracing" in t for t in titles) + assert "App Insights connection" in titles + assert "Agent tracing instrumentation" in titles cont_row = next( c for c in readiness["checks"] if "Continuous evaluation rules" in c["title"] @@ -452,7 +367,7 @@ def test_readiness_splits_tracing_and_includes_continuous_eval(tmp_path: Path): html = _render_readiness_section(readiness) assert "&rarr;" not in html - assert "Server-side tracing (agent → App Insights)" in html + assert "App Insights connection" in html def test_readiness_detects_multiturn_rubric_sampling_and_replay(tmp_path: Path): @@ -492,6 +407,114 @@ def test_readiness_detects_multiturn_rubric_sampling_and_replay(tmp_path: Path): assert by_title["Trace replay linked to evidence"]["status"] == "ok" +def test_readiness_detects_hosted_otel_eval_rubric_and_unknown_alerts( + tmp_path: Path, +): + from agentops.agent.cockpit import _build_readiness_checklist + + (tmp_path / "azure.yaml").write_text( + "services:\n" + " helpdeskbot:\n" + " host: azure.ai.agent\n" + " kind: hosted\n" + " project: ./src/helpdeskbot\n", + encoding="utf-8", + ) + source = tmp_path / "src" / "helpdeskbot" + source.mkdir(parents=True) + (source / "acs_middleware.py").write_text( + "from opentelemetry import trace\n" + "tracer = trace.get_tracer(__name__)\n" + "with tracer.start_as_current_span('acs'):\n" + " pass\n", + encoding="utf-8", + ) + (source / "eval.yaml").write_text( + "evaluators:\n" + " - name: helpdeskbot-safe-eval\n" + " local_uri: evaluators/helpdeskbot-safe-eval\n", + encoding="utf-8", + ) + + readiness = _build_readiness_checklist( + tmp_path, + { + "enabled": True, + "detail": "Linked through Project Managed Identity.", + "portal_url": "https://portal.azure.com/#resource/appi", + }, + {"has_data": False}, + watchdog=None, + ) + by_title = {check["title"]: check for check in readiness["checks"]} + + assert by_title["App Insights connection"]["status"] == "ok" + tracing = by_title["Agent tracing instrumentation"] + assert tracing["status"] == "ok" + assert "native tracing" in tracing["detail"] + assert "no application-side OpenTelemetry setup is required" in tracing["detail"] + assert "acs_middleware.py" in tracing["detail"] + assert by_title["Optional rubric evaluator gate"]["status"] == "ok" + assert "src/helpdeskbot/eval.yaml" in by_title[ + "Optional rubric evaluator gate" + ]["detail"] + assert by_title["Alerts wired"]["status"] == "info" + assert "Not verified" in by_title["Alerts wired"]["detail"] + assert "does not claim" in by_title["Alerts wired"]["detail"] + + +def test_readiness_recognizes_prompt_agent_native_tracing(tmp_path: Path): + from agentops.agent.cockpit import _build_readiness_checklist + + (tmp_path / "agentops.yaml").write_text( + "version: 1\n" + "agent: support-agent:4\n" + "dataset: .agentops/data/smoke.jsonl\n", + encoding="utf-8", + ) + + readiness = _build_readiness_checklist( + tmp_path, + {"enabled": True, "detail": "Linked", "portal_url": "https://x"}, + {"has_data": False}, + watchdog=None, + ) + tracing = next( + check + for check in readiness["checks"] + if check["title"] == "Agent tracing instrumentation" + ) + + assert tracing["status"] == "ok" + assert "prompt agent runtime" in tracing["detail"] + assert "Custom spans remain optional" in tracing["detail"] + + +def test_readiness_detects_alerts_declared_as_infrastructure(tmp_path: Path): + from agentops.agent.cockpit import _build_readiness_checklist + + infra = tmp_path / "infra" + infra.mkdir() + (infra / "alerts.bicep").write_text( + "resource failedRequests 'Microsoft.Insights/metricAlerts@2018-03-01' = {\n" + " name: 'failed-requests'\n" + "}\n", + encoding="utf-8", + ) + + readiness = _build_readiness_checklist( + tmp_path, + {"enabled": True, "detail": "Linked", "portal_url": "https://x"}, + {"has_data": False}, + watchdog=None, + ) + alerts = next( + check for check in readiness["checks"] if check["title"] == "Alerts wired" + ) + assert alerts["status"] == "ok" + assert "infra/alerts.bicep" in alerts["detail"] + + def test_readiness_non_ready_items_include_remediation(tmp_path: Path, monkeypatch): from agentops.agent.cockpit import _build_readiness_checklist @@ -514,13 +537,16 @@ def test_readiness_non_ready_items_include_remediation(tmp_path: Path, monkeypat assert non_ready for check in non_ready: detail = check["detail"] + if check["title"] == "Alerts wired": + assert "Not verified" in detail + continue assert "How to complete:" in detail assert ("" in detail) or ("Foundry" in detail) by_title = {check["title"]: check["detail"] for check in readiness["checks"]} - assert "OpenTelemetry" in by_title["Client-side tracing (app code instrumented)"] + assert "OpenTelemetry" in by_title["Agent tracing instrumentation"] assert "agentops eval run" in by_title["Scheduled eval (drift watch)"] assert "safe_agent_baseline.yaml" in by_title["Red team scans"] - assert "agentops.eval.*" in by_title["Alerts wired"] + assert "does not claim" in by_title["Alerts wired"] def test_readiness_dots_are_binary_ready_or_not(tmp_path: Path): @@ -580,45 +606,49 @@ def test_readiness_continuous_eval_warns_when_doctor_flags_missing_rules( assert "Foundry monitor docs" in cont_row["detail"] -def test_next_actions_do_not_suggest_workflow_when_ci_gate_exists(tmp_path: Path): - from agentops.agent.cockpit import ( - _build_next_actions, - _build_readiness_checklist, - ) +def test_next_actions_prioritize_doctor_then_incomplete_readiness(): + from agentops.agent.cockpit import _build_next_actions - workflows = tmp_path / ".github" / "workflows" - workflows.mkdir(parents=True) - (workflows / "agentops-pr.yml").write_text( - "name: AgentOps PR\nsteps:\n - run: agentops eval run\n", - encoding="utf-8", - ) - readiness = _build_readiness_checklist( - tmp_path, - {"enabled": True, "detail": "Linked", "portal_url": "https://x"}, - {"has_data": False}, + actions = _build_next_actions( watchdog={ - "has_history": True, "latest_findings": [ { - "id": "safety.config.continuous_eval_missing", + "id": "quality.answer", + "severity": "critical", + "title": "Answer quality is blocked", + "summary": "The response is incomplete.", + "recommendation": "Fix the response policy.", + }, + { + "id": "reliability.trace", "severity": "warning", - } + "title": "Trace coverage is incomplete", + "summary": "A trace is missing.", + "recommendation": "Enable trace capture.", + }, + ], + }, + readiness={ + "checks": [ + { + "title": "Server-side tracing", + "status": "warn", + "detail": "How to complete: enable tracing.", + }, + { + "title": "Alerts wired", + "status": "ok", + "detail": "Ready.", + }, ], }, ) - actions = _build_next_actions( - tmp_path, - {"enabled": True}, - watchdog={"latest_findings": []}, - readiness=readiness, - eval_payload={"runs": [object()]}, - ) - - assert not any( - action["title"] == "Add a CI eval workflow" - for action in actions["actions"] - ) + assert [action["title"] for action in actions["actions"]] == [ + "Fix Doctor: Answer quality is blocked", + "Fix Doctor: Trace coverage is incomplete", + "Complete readiness: Server-side tracing", + ] def test_readiness_detects_official_eval_workflow_and_evidence(tmp_path: Path): @@ -780,43 +810,6 @@ def test_readiness_details_include_azd_eval_and_governance_evidence(tmp_path: Pa assert "Governance evidence: assert: present, acs: present." in detail -def test_cockpit_surfaces_official_eval_artifacts_without_local_runs(tmp_path: Path): - official_dir = tmp_path / ".agentops" / "official-eval" - official_dir.mkdir(parents=True) - (official_dir / "metadata.json").write_text( - json.dumps( - { - "runner": "official-ai-agent-evaluation", - "items_total": 2, - "machine_readable_thresholds": False, - } - ), - encoding="utf-8", - ) - (official_dir / "result.json").write_text( - json.dumps( - { - "runner": "official-ai-agent-evaluation", - "status": "success", - "system": "github-actions", - "machine_readable_thresholds": False, - } - ), - encoding="utf-8", - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert payload["eval"]["has_runs"] is False - assert payload["eval"]["official_eval"]["present"] is True - assert "Official Microsoft Foundry AI Agent Evaluation evidence exists" in html - assert "agentops doctor --evidence-pack" in html - action_titles = [action["title"] for action in payload["next_actions"]["actions"]] - assert "Run your first evaluation" not in action_titles - assert "Generate release evidence" in action_titles - - def test_readiness_detects_prompt_agent_deploy_workflow(tmp_path: Path): from agentops.agent.cockpit import _build_readiness_checklist @@ -866,70 +859,6 @@ def test_readiness_continuous_eval_ok_when_doctor_finds_no_problem( assert cont_row["status"] == "ok" -def test_production_section_is_a_teaser_into_foundry_monitor(tmp_path: Path, monkeypatch): - """Production telemetry now ships as a 2-card teaser (error rate + - P95 latency) with a prominent "Full view in Foundry Monitor" CTA. The - cockpit keeps the quick health snapshot next to the full Foundry - Monitor drilldown.""" - _set_appinsights_env(monkeypatch) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - # Section title reflects the new "signal" framing. - assert "Production signal" in html - assert "Production telemetry" not in html - assert "Fast health snapshot from App Insights" in html - # The skeleton placeholder for the deferred load shows only the - # two surviving teaser cards. - assert "Error rate" in html - assert "P95 latency" in html - assert "Invocations" not in html or "Invocations" in ( - # The token "Invocations" may still appear in unrelated copy - # (e.g. App Insights description in the Foundry connection - # card). Restrict the assertion to the production grid. - "" - ) - - # The production grid renders exactly two skeleton placeholder cards - # (Error rate / P95 latency). Invocations and tokens stay in the full - # Foundry Monitor drilldown. - # The ``skeleton-card`` class is unique to the deferred production - # grid, so counting it across the full HTML is sufficient. - assert html.count("skeleton-card") == 2 - - -def test_production_section_links_to_foundry_monitor_first(tmp_path: Path, monkeypatch): - """The Production signal section must surface the Foundry Monitor - deep-link as a primary CTA so users always know where the full - runtime monitoring surface lives.""" - _set_appinsights_env(monkeypatch) - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-11T01-00-00Z", - passed=True, - metrics={"coherence": 5.0}, - cloud_evaluation={ - "eval_id": "evl_x", - "run_id": "run_x", - "report_url": ( - "https://acct.services.ai.azure.com/api/projects/p/" - "build/evaluations/evl_x" - ), - }, - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert "Full view in Foundry Monitor" in html - # The duplicated "Open App Insights KQL" CTA was removed from the - # Production signal section; the App Insights portal URL now lives only - # in the launchpad "App Insights" tile and the Doctor / Eval gate links. - assert "Open App Insights KQL" not in html - # The Foundry Monitor link is styled as the primary section CTA. - assert "section-link-primary" in html - - - def test_deployments_diagnostic_not_a_git_repo(tmp_path: Path): """Empty tempdir → deployments section explains it is not a git repo.""" from agentops.agent.cockpit import ( @@ -998,17 +927,8 @@ def test_create_app_serves_cockpit(tmp_path: Path): assert r.status_code == 200 assert "text/html" in r.headers["content-type"] assert "AgentOps Cockpit" in r.text - # Range bar is present. - assert "range-pills" in r.text - assert 'range=1d' in r.text and 'range=7d' in r.text and 'range=30d' in r.text - - # Custom range round-trip (also returns shell unless _partial is set). - r = client.get("/?range=custom&from=2020-01-01&to=2030-01-01&_partial=1") - assert r.status_code == 200 - - # Unknown range falls back to 7d default. - r = client.get("/?range=eternity&_partial=1") - assert r.status_code == 200 + assert "range-pills" not in r.text + assert "refreshSelect" not in r.text r = client.get("/healthz") assert r.status_code == 200 @@ -1218,68 +1138,6 @@ def test_foundry_deeplinks_use_only_build_routes(tmp_path): assert links["operate"].split("?")[0].endswith("/operate/overview") -def test_foundry_dataset_card_explains_inline_eval_data(tmp_path: Path): - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-12T22-30-00Z", - passed=True, - metrics={"similarity": 0.9}, - cloud_evaluation={ - "report_url": ( - "https://ai.azure.com/nextgen/r/" - "abc123,rg-x,,acct-y,proj-z/build/evaluations/" - "eval_001/run/run_001" - ), - "dataset": { - "mode": "inline", - "requested_mode": "auto", - "source_type": "file_content", - "local_path": ".agentops/data/smoke.jsonl", - "foundry_behavior": ( - "Foundry may materialize inline rows as eval-data-* " - "backing dataset assets." - ), - }, - }, - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert "Latest cloud run used local JSONL inline" in html - assert "eval-data-*" in html - - -def test_foundry_dataset_card_shows_synced_dataset(tmp_path: Path): - _write_eval_run( - tmp_path, - timestamp_dir="2026-05-12T22-31-00Z", - passed=True, - metrics={"similarity": 0.9}, - cloud_evaluation={ - "report_url": ( - "https://ai.azure.com/nextgen/r/" - "abc123,rg-x,,acct-y,proj-z/build/evaluations/" - "eval_001/run/run_001" - ), - "dataset": { - "mode": "foundry", - "requested_mode": "auto", - "source_type": "file_id", - "local_path": ".agentops/data/smoke.jsonl", - "foundry_name": "agentops-smoke", - "foundry_version": "sha256-abc123", - }, - }, - ) - - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - - assert "Latest cloud run used Foundry dataset agentops-smoke@sha256-abc123" in html - assert "eval-data-*" not in html - - def test_readiness_detail_links_use_info_color(tmp_path: Path, monkeypatch): monkeypatch.delenv("APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) monkeypatch.delenv("AGENTOPS_APPLICATIONINSIGHTS_CONNECTION_STRING", raising=False) @@ -1303,23 +1161,6 @@ def test_doctor_section_has_no_foundry_control_plane_link(tmp_path): assert "Open Foundry control plane" not in html -def test_tenant_card_source_moves_to_tooltip(tmp_path, monkeypatch): - """The "(from az account show)" suffix used to live inline in the - tenant card detail. It is now an ``(i)`` hover tooltip so the card - body stays compact.""" - # Force the tenant detection to a known value without invoking az. - monkeypatch.setattr( - "agentops.agent.cockpit._az_tenant_id", - lambda: "16b3c013-d300-468d-ac64-7eda0820b6d3", - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - # Inline source text is gone. - assert "(from az account show)" not in html - # Tooltip surfaces the same source via title= attribute. - assert "Resolved from `az account show`." in html - - def test_tenant_lookup_allows_slow_az_cmd_cold_start(monkeypatch): """Windows az.cmd can take several seconds on the first call. The Cockpit should wait long enough to resolve the tenant instead of @@ -1347,26 +1188,6 @@ def fake_run(*args, **kwargs): cockpit._TENANT_CACHE.clear() -def test_app_insights_card_source_moves_to_tooltip( - tmp_path, monkeypatch -): - """The "Connected via APPLICATIONINSIGHTS_CONNECTION_STRING" detail - used to render inline. It is now an ``(i)`` hover tooltip on the - App Insights card.""" - monkeypatch.setenv( - "APPLICATIONINSIGHTS_CONNECTION_STRING", - "InstrumentationKey=00000000-0000-0000-0000-000000000000;" - "IngestionEndpoint=https://example.in.applicationinsights.azure.com/", - ) - payload = build_cockpit_payload(tmp_path, time_range=_WIDE) - html = render_cockpit_html(payload) - # The verbose inline message is gone. - assert "Connected via APPLICATIONINSIGHTS_CONNECTION_STRING" not in html - # The tooltip carries the env-var reference. - assert "APPLICATIONINSIGHTS_CONNECTION_STRING" in html - assert "info-i" in html - - def test_app_insights_logs_query_is_bounded(monkeypatch): from agentops.agent.cockpit import _appinsights_portal_url @@ -1439,7 +1260,7 @@ def test_app_insights_eval_runs_query_and_link(monkeypatch, tmp_path): payload = build_cockpit_payload(tmp_path, time_range=_WIDE) html = render_cockpit_html(payload) - assert "View CI evals in App Insights" in html + assert "View CI evals in App Insights" not in html def test_foundry_project_card_compacts_endpoint_and_exposes_copy(tmp_path, monkeypatch): diff --git a/tests/unit/test_dashboard.py b/tests/unit/test_dashboard.py deleted file mode 100644 index 8bca286..0000000 --- a/tests/unit/test_dashboard.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Unit tests for the Foundry operations dashboard service. - -These cover the pure, Azure-free surface of -:mod:`agentops.services.dashboard`: template loading, the portal URL builder -(deployed vs. gallery fallback), the diagnostic-settings helpers, the ARM -template shape, and the RBAC preflight messaging with the authorization -helpers mocked. -""" - -from __future__ import annotations - -import json -from importlib.resources import files as package_files -from pathlib import Path - -import pytest - -from agentops.services import dashboard as dash - - -_FIXTURES = Path(__file__).parents[1] / "fixtures" -_EVALUATOR_KEYS = ("gen_ai.evaluation.name", "evaluator") -_SCORE_KEYS = ( - "gen_ai.evaluation.score.value", - "gen_ai.evaluation.score", - "score", -) -_LABEL_KEYS = ( - "gen_ai.evaluation.score.label", - "gen_ai.evaluation.result", - "label", -) - - -def _first_property(properties: dict[str, object], keys: tuple[str, ...]) -> str: - for key in keys: - value = properties.get(key) - if value not in (None, ""): - return str(value) - return "" - - -def _fixture_projection(event: dict[str, object]) -> dict[str, object]: - properties = event["properties"] - assert isinstance(properties, dict) - evaluator = _first_property(properties, _EVALUATOR_KEYS) - score_text = _first_property(properties, _SCORE_KEYS) - label = _first_property(properties, _LABEL_KEYS) - try: - numeric_score = float(score_text) if score_text else None - except ValueError: - numeric_score = None - agent_id = str(properties.get("gen_ai.agent.id", "")) - version = str(properties.get("gen_ai.agent.version", "")) - if not version and ":" in agent_id: - version = agent_id.split(":", 1)[1] - return { - "recognized": bool(evaluator or score_text or label), - "numeric_score": numeric_score, - "version": version or "Version not reported", - "raw_properties": properties, - } - - -# --------------------------------------------------------------------------- -# Template loading -# --------------------------------------------------------------------------- -def test_load_workbook_template_returns_valid_json() -> None: - raw = dash.load_workbook_template() - assert isinstance(raw, str) and raw.strip() - parsed = json.loads(raw) - # The packaged asset is a gallery-template workbook. - assert isinstance(parsed, dict) - - -def test_load_workbook_content_matches_template() -> None: - assert dash.load_workbook_content() == json.loads(dash.load_workbook_template()) - - -def test_agent_behavior_tab_is_additive_and_preserves_existing_navigation() -> None: - content = dash.load_workbook_content() - tabs = next(item for item in content["items"] if item["name"] == "tabs") - labels = [link["linkLabel"] for link in tabs["content"]["links"]] - assert labels == [ - "Capacity", - "Traffic and tokens", - "Latency", - "Errors and throttling", - "Agent behavior", - ] - groups = {item["name"]: item for item in content["items"] if item["type"] == 12} - assert { - "group-capacity", - "group-traffic", - "group-latency", - "group-errors", - }.issubset(groups) - behavior = groups["group-agent-behavior"] - assert behavior["conditionalVisibility"]["value"] == "agent-behavior" - - -def test_agent_behavior_tab_surfaces_states_filters_and_preview_boundary() -> None: - content = dash.load_workbook_content() - behavior = next( - item for item in content["items"] if item["name"] == "group-agent-behavior" - ) - items = behavior["content"]["items"] - note = next(item for item in items if item["name"] == "agent-behavior-note") - note_text = note["content"]["json"] - for state in ( - "Schema unavailable", - "No access", - "No data", - "Filter empty", - "Possible ingestion delay", - ): - assert state in note_text - assert "Preview" in note_text - assert "Foundry" in note_text - assert "does not create, schedule, gate, or edit evaluations" in note_text - assert "human trace annotations" in note_text - assert "validation-dependent" in note_text - assert "does not require `gen_ai.agent.id`" in note_text - - filters = next(item for item in items if item["name"] == "agent-behavior-filters") - assert [parameter["name"] for parameter in filters["content"]["parameters"]] == [ - "AgentEnvironment", - "AgentName", - "AgentVersion", - "Evaluator", - ] - - -def test_agent_behavior_queries_use_bounded_versioned_normalization() -> None: - content = dash.load_workbook_content() - behavior = next( - item for item in content["items"] if item["name"] == "group-agent-behavior" - ) - query_items = [ - item - for item in behavior["content"]["items"] - if item["type"] == 3 and "query" in item["content"] - ] - assert len(query_items) == 7 - assert all( - item["content"]["timeContextFromParameter"] == "TimeRange" - for item in query_items - ) - combined = "\n".join(item["content"]["query"] for item in query_items) - for fragment in ( - "set best_effort=true", - "union isfuzzy=true", - "AppEvents", - "customEvents", - "Name == 'gen_ai.evaluation.result'", - "name == 'gen_ai.evaluation.result'", - "Properties", - "customDimensions", - "gen_ai.evaluation.score.value", - "gen_ai.evaluation.score.label", - "Version not reported", - "RawProperties", - ): - assert fragment in combined - status = next( - item for item in query_items if item["name"] == "agent-behavior-status" - ) - status_query = status["content"]["query"] - assert "agent_behavior/v1" in status_query - assert "ObservedInvokeAgentInvocations" in status_query - assert "EvaluatedTraces" in status_query - assert "EvaluationEvents" in status_query - assert "Coverage" not in status_query - assert "automated trace-evaluation export validation-dependent" in status_query - schema_diagnostics = next( - item - for item in query_items - if item["name"] == "agent-behavior-schema-diagnostics" - ) - assert "RawProperties" in schema_diagnostics["content"]["query"] - assert "| take 100" in schema_diagnostics["content"]["query"] - - -def test_agent_behavior_does_not_combine_unlike_raw_score_scales() -> None: - content = dash.load_workbook_content() - behavior = next( - item for item in content["items"] if item["name"] == "group-agent-behavior" - ) - items = {item["name"]: item for item in behavior["content"]["items"]} - assert items["agent-behavior-score-trend"]["content"]["visualization"] == "table" - assert items["agent-behavior-pass-trend"]["content"]["visualization"] == "timechart" - assert ( - items["agent-behavior-volume-trend"]["content"]["visualization"] == "timechart" - ) - assert ( - "do not compare evaluators" - in items["agent-behavior-score-trend"]["content"]["title"] - ) - - -def test_agent_behavior_authoring_query_is_packaged_and_bounded() -> None: - resource = ( - package_files("agentops.templates") - .joinpath("workbooks/queries/agent_behavior.kql") - .read_text(encoding="utf-8") - ) - assert "agent_behavior/v1" in resource - assert "between (_startTime .. _endTime)" in resource - assert "AppEvents" in resource and "customEvents" in resource - assert "AppDependencies" in resource and "dependencies" in resource - assert "RawProperties" in resource - assert "human trace" in resource and "annotations" in resource - assert "Automated trace-evaluation export" in resource - assert "validation-dependent" in resource - - -def test_agent_behavior_schema_fixtures_cover_supported_shapes_and_edges() -> None: - events = json.loads( - (_FIXTURES / "workbook_agent_behavior_events.json").read_text(encoding="utf-8") - ) - assert {event["source_table"] for event in events} == { - "AppEvents", - "customEvents", - } - expected_columns = { - "AppEvents": ("Name", "Properties"), - "customEvents": ("name", "customDimensions"), - } - projections = {} - for event in events: - name_column, properties_column = expected_columns[event["source_table"]] - assert event["name_column"] == name_column - assert event["properties_column"] == properties_column - assert event["event_name"] == "gen_ai.evaluation.result" - projection = _fixture_projection(event) - projections[event["case"]] = projection - assert projection["recognized"] is event["expected"]["recognized"] - assert projection["numeric_score"] == event["expected"]["numeric_score"] - assert projection["version"] == event["expected"]["version"] - assert projection["raw_properties"] == event["properties"] - - assert projections["missing_optional_fields"]["version"] == "Version not reported" - assert projections["nonnumeric_score"]["numeric_score"] is None - assert projections["unrecognized_schema"]["recognized"] is False - evaluators = { - _first_property(event["properties"], _EVALUATOR_KEYS) - for event in events - if event["expected"]["recognized"] - } - assert {"Relevance", "Groundedness", "IntentResolution", "Fluency"} <= evaluators - same_trace = [ - event for event in events if event["properties"].get("trace_id") == "trace-2" - ] - assert { - _first_property(event["properties"], _EVALUATOR_KEYS) for event in same_trace - } == {"Groundedness", "Fluency"} - - -# --------------------------------------------------------------------------- -# Portal URL -# --------------------------------------------------------------------------- -def test_make_workbook_resource_id_is_deterministic() -> None: - a = dash.make_workbook_resource_id("sub", "rg", "AgentOps Foundry operations") - b = dash.make_workbook_resource_id("sub", "rg", "AgentOps Foundry operations") - assert a == b - assert a.startswith("/subscriptions/sub/resourceGroups/rg/providers/") - assert dash.WORKBOOK_RESOURCE_TYPE in a - - -def test_portal_url_deployed_when_target_known() -> None: - url = dash.build_workbook_portal_url( - subscription_id="sub", - resource_group="rg", - name="AgentOps Foundry operations", - tenant_id="tenant-123", - ) - assert url.startswith("https://portal.azure.com/#@tenant-123/resource") - assert url.endswith("/workbook") - - -def test_portal_url_deployed_without_tenant() -> None: - url = dash.build_workbook_portal_url( - subscription_id="sub", resource_group="rg", name="wb" - ) - assert url.startswith("https://portal.azure.com/#/resource") - assert "@" not in url.split("/resource", 1)[0] - - -def test_portal_url_falls_back_to_gallery_when_target_unknown() -> None: - url = dash.build_workbook_portal_url() - assert "WorkbookMenuBlade" in url - assert url.endswith("/gallery") - - -def test_portal_url_prefers_explicit_resource_id() -> None: - rid = "/subscriptions/s/resourceGroups/g/providers/x/y/z" - url = dash.build_workbook_portal_url(workbook_resource_id=rid, tenant_id="t") - assert url == f"https://portal.azure.com/#@t/resource{rid}/workbook" - - -# --------------------------------------------------------------------------- -# Diagnostic settings -# --------------------------------------------------------------------------- -def test_missing_diagnostic_categories_none_when_all_present() -> None: - assert ( - dash.missing_diagnostic_categories( - ["RequestResponse", "AzureOpenAIRequestUsage", "Audit"] - ) - == [] - ) - - -def test_missing_diagnostic_categories_reports_absent() -> None: - assert dash.missing_diagnostic_categories(["RequestResponse"]) == [ - "AzureOpenAIRequestUsage" - ] - assert dash.missing_diagnostic_categories([]) == list( - dash.REQUIRED_DIAGNOSTIC_CATEGORIES - ) - - -def test_build_diagnostic_settings_command_shape() -> None: - cmd = dash.build_diagnostic_settings_command( - aoai_resource_id="/subscriptions/s/aoai", - workspace_id="/subscriptions/s/ws", - ) - assert cmd.startswith("az monitor diagnostic-settings create ") - assert "--resource /subscriptions/s/aoai" in cmd - assert "--workspace /subscriptions/s/ws" in cmd - assert "RequestResponse" in cmd and "AzureOpenAIRequestUsage" in cmd - - -def test_build_diagnostic_settings_command_uses_placeholders() -> None: - cmd = dash.build_diagnostic_settings_command( - aoai_resource_id=None, workspace_id=None - ) - assert "" in cmd - assert "" in cmd - - -# --------------------------------------------------------------------------- -# ARM template -# --------------------------------------------------------------------------- -def test_build_arm_template_shape() -> None: - target = dash.DashboardTarget( - name="AgentOps Foundry operations", - subscription_id="sub", - resource_group="rg", - workspace_id="/subscriptions/sub/ws", - ) - template = dash.build_arm_template(target=target) - assert template["$schema"].startswith("https://schema.management.azure.com") - assert len(template["resources"]) == 1 - resource = template["resources"][0] - assert resource["type"] == dash.WORKBOOK_RESOURCE_TYPE - assert resource["properties"]["displayName"] == "AgentOps Foundry operations" - # serializedData is the workbook content re-serialized as a string. - assert json.loads(resource["properties"]["serializedData"]) == ( - dash.load_workbook_content() - ) - assert resource["properties"]["sourceId"] == "/subscriptions/sub/ws" - - -def test_build_arm_template_location_defaults_to_resource_group_expression() -> None: - # Workbooks reject "global"; the template must resolve to the RG region. - target = dash.DashboardTarget(name="wb", subscription_id="sub", resource_group="rg") - template = dash.build_arm_template(target=target) - assert template["resources"][0]["location"] == "[resourceGroup().location]" - - -def test_build_arm_template_location_honors_explicit_region() -> None: - target = dash.DashboardTarget( - name="wb", subscription_id="sub", resource_group="rg", location="eastus2" - ) - template = dash.build_arm_template(target=target) - assert template["resources"][0]["location"] == "eastus2" - - -def test_build_arm_template_name_matches_resource_id() -> None: - target = dash.DashboardTarget(name="wb", subscription_id="sub", resource_group="rg") - template = dash.build_arm_template(target=target) - rid = dash.make_workbook_resource_id("sub", "rg", "wb") - assert template["resources"][0]["name"] == rid.rsplit("/", 1)[-1] - - -# --------------------------------------------------------------------------- -# RBAC preflight -# --------------------------------------------------------------------------- -def test_check_rbac_errors_without_subscription() -> None: - result = dash.check_rbac( - subscription_id=None, resource_group="rg", workspace_id=None - ) - assert result.ok is False - assert result.level == "error" - assert any("subscription" in m.lower() for m in result.messages) - - -def _patch_rbac(monkeypatch, *, rg_roles, ws_roles=None, raise_error=None): - """Install fakes for the lazy-imported authorization helpers.""" - import agentops.agent.checks._rbac_authorization as rbac_mod - - class _AuthErr(Exception): - pass - - monkeypatch.setattr(rbac_mod, "AuthorizationCheckError", _AuthErr) - monkeypatch.setattr( - rbac_mod, "resolve_signed_in_principal_object_id", lambda: "principal-oid" - ) - - def _list(*, subscription_id, scope, principal_object_id): - if raise_error is not None: - raise _AuthErr(raise_error) - if scope.endswith("/resourceGroups/rg"): - return list(rg_roles) - return list(ws_roles or []) - - monkeypatch.setattr(rbac_mod, "list_principal_role_definition_ids", _list) - return _AuthErr - - -def test_check_rbac_passes_when_roles_present(monkeypatch) -> None: - _patch_rbac( - monkeypatch, - rg_roles=[dash._ROLE_WORKBOOK_CONTRIBUTOR], - ws_roles=[dash._ROLE_LOG_ANALYTICS_READER], - ) - result = dash.check_rbac( - subscription_id="sub", - resource_group="rg", - workspace_id="/subscriptions/sub/ws", - ) - assert result.ok is True - assert result.level == "ok" - assert any("passed" in m.lower() for m in result.messages) - - -def test_check_rbac_fails_when_workbook_role_missing(monkeypatch) -> None: - _patch_rbac(monkeypatch, rg_roles=[dash._ROLE_READER], ws_roles=[]) - result = dash.check_rbac( - subscription_id="sub", resource_group="rg", workspace_id=None - ) - assert result.ok is False - assert result.level == "error" - joined = " ".join(result.messages) - assert "Workbook Contributor" in joined - assert "'rg'" in joined - assert "--dry-run" in joined - - -def test_check_rbac_fails_when_workspace_role_missing(monkeypatch) -> None: - _patch_rbac( - monkeypatch, - rg_roles=[dash._ROLE_WORKBOOK_CONTRIBUTOR], - ws_roles=[dash._ROLE_READER.replace("a", "z")], # unrelated role guid - ) - result = dash.check_rbac( - subscription_id="sub", - resource_group="rg", - workspace_id="/subscriptions/sub/ws", - ) - assert result.ok is False - assert any("Log Analytics Reader" in m for m in result.messages) - - -def test_check_rbac_fails_open_on_authorization_error(monkeypatch) -> None: - _patch_rbac(monkeypatch, rg_roles=[], raise_error="listing denied") - result = dash.check_rbac( - subscription_id="sub", resource_group="rg", workspace_id=None - ) - # Fails OPEN: warn but allow deploy to proceed. - assert result.ok is True - assert result.level == "warn" - assert any("listing denied" in m for m in result.messages) - - -def test_check_rbac_fails_open_when_helpers_unavailable(monkeypatch) -> None: - import builtins - - real_import = builtins.__import__ - - def _blocked(name, *args, **kwargs): - if name == "agentops.agent.checks._rbac_authorization": - raise ImportError("no authorization helpers") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", _blocked) - result = dash.check_rbac( - subscription_id="sub", resource_group="rg", workspace_id=None - ) - assert result.ok is True - assert result.level == "warn" - - -# --------------------------------------------------------------------------- -# deploy_workbook guards (no live Azure) -# --------------------------------------------------------------------------- -def test_deploy_workbook_requires_subscription() -> None: - target = dash.DashboardTarget(name="wb") - with pytest.raises(dash.DashboardError) as exc: - dash.deploy_workbook(target=target) - assert "subscription" in str(exc.value).lower() - - -def test_deploy_workbook_errors_when_az_missing(monkeypatch) -> None: - monkeypatch.setattr(dash, "_az_executable", lambda: None) - target = dash.DashboardTarget(name="wb", subscription_id="sub", resource_group="rg") - with pytest.raises(dash.DashboardError) as exc: - dash.deploy_workbook(target=target) - assert "az" in str(exc.value).lower() diff --git a/tests/unit/test_foundry_discovery.py b/tests/unit/test_foundry_discovery.py index 235ba1a..c5896ab 100644 --- a/tests/unit/test_foundry_discovery.py +++ b/tests/unit/test_foundry_discovery.py @@ -272,6 +272,39 @@ def test_with_reason_accepts_project_managed_identity_connection(): assert reason == PROJECT_MANAGED_IDENTITY_APPINSIGHTS_REASON +def test_resource_id_discovery_accepts_project_managed_identity_metadata(): + resource_id = ( + "/subscriptions/000/resourceGroups/rg/providers/" + "Microsoft.Insights/components/appi" + ) + connection = mock.Mock() + connection.type = "ConnectionType.APPLICATION_INSIGHTS" + connection.target = resource_id + fake_connections = mock.MagicMock() + fake_connections.list.return_value = iter([connection]) + fake_client = mock.MagicMock() + fake_client.connections = fake_connections + fake_projects_mod = mock.MagicMock() + fake_projects_mod.AIProjectClient.return_value = fake_client + fake_identity_mod = mock.MagicMock() + + with mock.patch.dict( + "sys.modules", + {"azure.ai.projects": fake_projects_mod, "azure.identity": fake_identity_mod}, + ): + from agentops.utils.foundry_discovery import ( + resolve_appinsights_resource_id_with_reason, + ) + + result, reason = resolve_appinsights_resource_id_with_reason( + "https://x.services.ai.azure.com/api/projects/pmi" + ) + + assert result == resource_id + assert reason is None + fake_connections.list.assert_called_once_with() + + def test_with_reason_reports_missing_app_insights_connection(): class ResourceNotFoundError(Exception): pass diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index 926717e..fcb1c9c 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -581,6 +581,62 @@ def query_resource( assert payload.p95_duration_seconds == 2.5 +def test_azure_monitor_uses_foundry_app_insights_resource_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from agentops.utils import foundry_discovery + + resource_id = ( + "/subscriptions/000/resourceGroups/rg/providers/" + "Microsoft.Insights/components/appi-pmi" + ) + captured: dict[str, object] = {} + + class LogsQueryStatus: + FAILURE = "Failure" + + class Response: + status = "Success" + tables: list[object] = [] + + class LogsQueryClient: + def __init__(self, _credential: object) -> None: + pass + + def query_resource( + self, + *, + resource_id: str, + query: str, + timespan: object, + ) -> Response: + captured["resource_id"] = resource_id + return Response() + + identity_module = types.ModuleType("azure.identity") + identity_module.DefaultAzureCredential = object # type: ignore[attr-defined] + query_module = types.ModuleType("azure.monitor.query") + query_module.LogsQueryClient = LogsQueryClient # type: ignore[attr-defined] + query_module.LogsQueryStatus = LogsQueryStatus # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "azure.identity", identity_module) + monkeypatch.setitem(sys.modules, "azure.monitor.query", query_module) + monkeypatch.setattr( + foundry_discovery, + "resolve_appinsights_resource_id_from_env_with_reason", + lambda: (resource_id, None), + ) + + payload = azure_monitor.collect_azure_monitor( + AzureMonitorSourceConfig(enabled=True), + lookback_days=7, + ) + + assert captured["resource_id"] == resource_id + assert payload.diagnostics["target"] == resource_id + assert payload.diagnostics["target_source"] == "foundry_project_connection" + assert payload.diagnostics["status"] == "ok" + + def test_azure_monitor_uses_connection_string_application_id( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_telemetry_import.py b/tests/unit/test_telemetry_import.py deleted file mode 100644 index aa07010..0000000 --- a/tests/unit/test_telemetry_import.py +++ /dev/null @@ -1,153 +0,0 @@ -from __future__ import annotations - -import builtins -import json - -import pytest - -from agentops.core.agentops_config import AgentOpsConfig -from agentops.services.telemetry_import import ( - TelemetryImportError, - build_telemetry_kql, - find_telemetry_import, - query_azure_monitor, - transform_telemetry_rows, - write_telemetry_import, -) - - -def _config(**overrides): - data = { - "version": 1, - "agent": "support-agent:1", - "dataset": ".agentops/data/smoke.jsonl", - "telemetry_imports": [ - { - "name": "prod", - "target": "application-insights", - "resource_id": "$APPINSIGHTS_RESOURCE_ID", - "fields": { - "input": "customDimensions.question", - "response": "customDimensions.answer", - "context": "customDimensions.context", - }, - "output": {"path": ".agentops/data/prod.jsonl"}, - **overrides, - } - ], - } - return AgentOpsConfig.model_validate(data).telemetry_imports[0] - - -def test_transform_rows_dedupes_redacts_and_writes_manifest(tmp_path) -> None: - cfg = _config( - output={"path": str(tmp_path / "prod.jsonl")}, - privacy={"redact_fields": ["token"], "max_field_length": 100, "include_raw": True}, - ) - raw = [ - { - "operation_Id": "trace-1", - "id": "turn-1", - "customDimensions": { - "question": "How do I reset my password?", - "answer": "Open account settings.", - "context": "Reset article", - "token": "secret-token", - }, - }, - { - "operation_Id": "trace-1", - "id": "turn-1", - "customDimensions": { - "question": "How do I reset my password?", - "answer": "Open account settings.", - }, - }, - {"customDimensions": {"question": "missing response"}}, - ] - - preview = transform_telemetry_rows(cfg, raw) - write_telemetry_import(preview) - - assert len(preview.rows) == 1 - assert preview.deduped == 1 - assert preview.skipped == 1 - row = preview.rows[0] - assert row["input"] == "How do I reset my password?" - assert row["response"] == "Open account settings." - assert row["expected"] == "Open account settings." - assert row["context"] == "Reset article" - assert row["telemetry"]["trace_id"] == "trace-1" - assert row["raw"]["customDimensions"]["token"] == "[redacted]" - assert (tmp_path / "prod.jsonl").exists() - manifest = json.loads((tmp_path / "prod-manifest.json").read_text(encoding="utf-8")) - assert manifest["rows"] == 1 - assert manifest["deduped"] == 1 - - -def test_build_kql_uses_safe_generated_filters() -> None: - cfg = _config(filters={"customDimensions.agent": ["support", "sales"]}, max_rows=1000) - - kql = build_telemetry_kql(cfg, rows=5) - - assert "union isfuzzy=true requests, dependencies, traces" in kql - assert "| extend timestamp = coalesce(" in kql - assert "column_ifexists('timestamp', datetime(null))" in kql - assert "column_ifexists('TimeGenerated', datetime(null))" in kql - assert "coalesce(timestamp, TimeGenerated)" not in kql - assert "ago(7d)" in kql - assert ( - "tostring(column_ifexists('customDimensions', dynamic({}))['agent']) " - "in ('support', 'sales')" - ) in kql - assert "operation_Id = column_ifexists('operation_Id', '')" in kql - assert "TimeGenerated =" not in kql - assert "| order by timestamp desc" in kql - assert "take 5" in kql - - -def test_build_kql_guards_plain_filter_columns() -> None: - cfg = _config(filters={"name": "agent.response"}) - - kql = build_telemetry_kql(cfg, rows=10) - - assert "tostring(column_ifexists('name', '')) == 'agent.response'" in kql - assert "tostring(name)" not in kql - - -def test_build_kql_rejects_unsafe_filter_field() -> None: - cfg = _config(filters={"name); drop table traces; //": "x"}) - - with pytest.raises(TelemetryImportError, match="unsafe"): - build_telemetry_kql(cfg) - - -def test_find_telemetry_import_reports_available_names() -> None: - cfg = AgentOpsConfig.model_validate( - { - "version": 1, - "agent": "support-agent:1", - "dataset": ".agentops/data/smoke.jsonl", - "telemetry_imports": [ - {"name": "prod", "target": "log-analytics", "workspace_id": "workspace"} - ], - } - ) - - with pytest.raises(TelemetryImportError, match="prod"): - find_telemetry_import(cfg, "missing") - - -def test_query_azure_monitor_reports_missing_sdk(monkeypatch) -> None: - cfg = _config() - original_import = builtins.__import__ - - def fake_import(name, *args, **kwargs): - if name == "azure.identity": - raise ImportError("no azure") - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", fake_import) - - with pytest.raises(TelemetryImportError, match="azure-identity"): - query_azure_monitor(cfg, rows=1) From 456b2b2de4f2bb4d73749be5a1d605988d1d218e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:40:28 +0000 Subject: [PATCH 8/8] chore: prepare release 0.9.0 --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- CHANGELOG.md | 2 ++ plugins/agentops/package.json | 2 +- plugins/agentops/plugin.json | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c0c2781..ac349d6 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.8.8", + "version": "0.9.0", "keywords": [ "agentops", "evaluation", diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index c0c2781..ac349d6 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.8.8", + "version": "0.9.0", "keywords": [ "agentops", "evaluation", diff --git a/CHANGELOG.md b/CHANGELOG.md index eeac465..eb253b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +## [0.9.0] - 2026-08-14 + ### Changed - **The AgentOps Cockpit now focuses on five release-readiness sections.** The page renders only the readiness and Doctor ship verdicts, Foundry and diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index 1f51c25..d7639ec 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -2,7 +2,7 @@ "name": "agentops-accelerator", "displayName": "AgentOps Accelerator — Skills for GitHub Copilot", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.8.8", + "version": "0.9.0", "publisher": "AgentOpsAccelerator", "icon": "icon.png", "license": "MIT", diff --git a/plugins/agentops/plugin.json b/plugins/agentops/plugin.json index 30b24eb..abe32ba 100644 --- a/plugins/agentops/plugin.json +++ b/plugins/agentops/plugin.json @@ -1,7 +1,7 @@ { "name": "agentops-accelerator", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.8.8", + "version": "0.9.0", "author": { "name": "AgentOps Accelerator", "url": "https://github.com/Azure/agentops"