From c49922d19bd90d68f5d7dcc6e2df49974b8c5578 Mon Sep 17 00:00:00 2001 From: David Haifley Date: Thu, 17 Sep 2026 14:22:30 -0400 Subject: [PATCH 1/4] feat(tracing): add harness invocation identity, lifecycle, and bounded capture Signed-off-by: David Haifley --- docs/architecture/README.md | 1 + docs/architecture/telemetry.md | 132 ++++++ go/adk/pkg/a2a/server/server.go | 14 +- go/adk/pkg/a2a/server/tracing.go | 76 ++-- go/adk/pkg/a2a/server/tracing_test.go | 311 +++++++++++++ go/adk/pkg/app/app.go | 18 +- .../internal/translator/claude/compiler.go | 16 +- .../translator/claude/compiler_test.go | 62 +++ go/core/internal/translator/codex/compiler.go | 5 + .../translator/codex/compiler_test.go | 54 +++ go/core/internal/translator/otel.go | 37 +- go/core/test/e2e/tracing_test.go | 73 ++- go/harness/claude/cmd/main.go | 14 +- go/harness/claude/config/config.go | 19 +- go/harness/claude/config/config_test.go | 50 ++- go/harness/claude/executor/executor.go | 12 +- go/harness/claude/executor/executor_test.go | 5 +- go/harness/claude/internal/adapter/adapter.go | 4 + .../claude/internal/adapter/adapter_test.go | 2 +- go/harness/claude/internal/driver/process.go | 3 + go/harness/codex/cmd/main.go | 14 +- go/harness/codex/config/config.go | 19 +- go/harness/codex/config/config_test.go | 29 ++ go/harness/codex/executor/executor.go | 11 +- go/harness/codex/executor/executor_test.go | 5 +- go/harness/codex/internal/adapter/adapter.go | 4 + go/harness/codex/internal/driver/process.go | 6 + go/harness/runtime/a2a/executor.go | 176 +++++++- go/harness/runtime/a2a/executor_test.go | 21 +- go/harness/runtime/a2a/tracing_test.go | 414 ++++++++++++++++++ go/pkg/tracing/capture.go | 97 ++++ go/pkg/tracing/capture_test.go | 111 +++++ go/pkg/tracing/invocation.go | 168 +++++++ go/pkg/tracing/invocation_test.go | 176 ++++++++ go/pkg/tracing/runtime.go | 252 +++++++++++ go/pkg/tracing/runtime_test.go | 152 +++++++ go/pkg/tracing/tracing.go | 34 +- 37 files changed, 2490 insertions(+), 107 deletions(-) create mode 100644 docs/architecture/telemetry.md create mode 100644 go/adk/pkg/a2a/server/tracing_test.go create mode 100644 go/harness/runtime/a2a/tracing_test.go create mode 100644 go/pkg/tracing/capture.go create mode 100644 go/pkg/tracing/capture_test.go create mode 100644 go/pkg/tracing/invocation.go create mode 100644 go/pkg/tracing/invocation_test.go create mode 100644 go/pkg/tracing/runtime.go create mode 100644 go/pkg/tracing/runtime_test.go diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 39fb422804..f5303ad03b 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -76,6 +76,7 @@ quiescence. It reaches Actors through the private runtime network. - [A2A agent tools](a2a-subagents.md) - [Human in the loop](human-in-the-loop.md) - [Prompt resolution](prompt-templates.md) +- [Telemetry](telemetry.md) The documents describe implemented behavior. Deferred work, including full cross-AgentInstance delegation and Dedicated agents, belongs in the diff --git a/docs/architecture/telemetry.md b/docs/architecture/telemetry.md new file mode 100644 index 0000000000..bdb8502f16 --- /dev/null +++ b/docs/architecture/telemetry.md @@ -0,0 +1,132 @@ +# Telemetry + +Kagent exports OpenTelemetry traces from agent runtimes when a user enables +them. This document describes what a runtime produces, so consumers can rely on +it without reading runtime internals. + +## Enabling export + +The controller resolves telemetry from its own process environment and compiles +the result into each runtime revision. + +| Variable | Effect | +| --- | --- | +| `OTEL_TRACING_ENABLED` | Enables trace export for compiled runtimes | +| `OTEL_LOGGING_ENABLED` | Enables log export for compiled runtimes | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Destination, with the usual signal-specific overrides | +| `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` or `http/protobuf`, with signal-specific overrides | +| `KAGENT_OTEL_CAPTURE_SENSITIVE_CONTENT` | Enables bounded prompt and response capture. Off by default | +| `KAGENT_OTEL_CAPTURE_RAW_API_BODIES` | Enables native raw provider body logging. Off by default | +| `KAGENT_OTEL_MAX_CAPTURE_BYTES` | Bytes retained per captured prompt and per captured response. Defaults to 16 KiB, ceiling 64 KiB | + +An unusable capture budget is reported as a compilation warning and replaced by +the default, so an observability setting cannot invalidate an AgentTemplate. + +Other `OTEL_*` variables remain available for per-Harness tuning through +`Harness.spec.env`, including `OTEL_RESOURCE_ATTRIBUTES`. + +## The invocation span + +Each A2A `SendMessage` or `SendStreamingMessage` request opens one span named +`a2a.request` in the instrumentation scope +`github.com/kagent-dev/kagent/go/adk/pkg/a2a/server`. It represents one +execution segment and is the anchor consumers should read. Model and tool spans +come from the runtime itself and are descendants of it. + +| Attribute | Meaning | +| --- | --- | +| `a2a.method` | `SendMessage` or `SendStreamingMessage` | +| `kagent.harness.kind` | `claude` or `codex`. Absent for ADK agents, which are not native harnesses | +| `gen_ai.agent.name` | The compiled agent identity, `