From e7fd46f1b1c695463069039eefb12969f6b052f8 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Thu, 5 Nov 2026 13:39:22 +0000 Subject: [PATCH 1/6] feat(sdk-openai-agents): add the openai agents sdk integration A trace processor for @openai/agents that sends each agent run to telemetry.dev as one trace: agents, turns, model calls with token usage, tool calls, and the commands a sandbox agent ran, with their exit codes and the sandbox backend. instrumentSandboxClient records the provider's sandbox id so the provider's own sandbox events can join the trace. Only model-call spans carry usage; task, turn, and agent spans repeat the totals of their calls. The root of a trace ends once the trace and all of its spans have ended, because other processors can delay span ends past the trace end, and a run that throws never ends its trace. --- README.md | 37 +- packages/openai-agents/LICENSE | 21 + packages/openai-agents/README.md | 108 ++++ packages/openai-agents/package.json | 73 +++ packages/openai-agents/src/config.ts | 28 + packages/openai-agents/src/index.ts | 5 + packages/openai-agents/src/processor.ts | 599 +++++++++++++++++ packages/openai-agents/src/sandbox.ts | 47 ++ .../openai-agents/tests/openai-agents.test.ts | 602 ++++++++++++++++++ packages/openai-agents/tsconfig.json | 20 + packages/openai-agents/vite.config.ts | 29 + pnpm-lock.yaml | 171 ++++- release-please-config.json | 5 + 13 files changed, 1716 insertions(+), 29 deletions(-) create mode 100644 packages/openai-agents/LICENSE create mode 100644 packages/openai-agents/README.md create mode 100644 packages/openai-agents/package.json create mode 100644 packages/openai-agents/src/config.ts create mode 100644 packages/openai-agents/src/index.ts create mode 100644 packages/openai-agents/src/processor.ts create mode 100644 packages/openai-agents/src/sandbox.ts create mode 100644 packages/openai-agents/tests/openai-agents.test.ts create mode 100644 packages/openai-agents/tsconfig.json create mode 100644 packages/openai-agents/vite.config.ts diff --git a/README.md b/README.md index 6ff460b5..ac85de93 100644 --- a/README.md +++ b/README.md @@ -139,24 +139,25 @@ Streamed output is kept within fixed size limits that each provider package's RE Each integration links to its installation instructions and API coverage. The link text is the npm or PyPI package name. -| Integration | TypeScript | Python | -| ---------------------------- | ------------------------------------------------------ | -------------------------------------------------------- | -| Core SDK | [`@telemetry-dev/sdk`](packages/sdk) | [`telemetry-dev`](sdks/python) | -| Existing OpenTelemetry setup | [`@telemetry-dev/otel`](packages/otel) | [`telemetry-dev`](sdks/python) (`telemetry_dev.otel`) | -| OpenAI | [`@telemetry-dev/openai`](packages/openai) | [`telemetry-dev-openai`](sdks/python-openai) | -| Anthropic | [`@telemetry-dev/anthropic`](packages/anthropic) | [`telemetry-dev-anthropic`](sdks/python-anthropic) | -| Amazon Bedrock | [`@telemetry-dev/bedrock`](packages/bedrock) | [`telemetry-dev-bedrock`](sdks/python-bedrock) | -| Google GenAI | [`@telemetry-dev/google-genai`](packages/google-genai) | [`telemetry-dev-google-genai`](sdks/python-google-genai) | -| OpenRouter | [`@telemetry-dev/openrouter`](packages/openrouter) | [`telemetry-dev-openrouter`](sdks/python-openrouter) | -| Vercel AI SDK | [`@telemetry-dev/ai-sdk`](packages/ai) | | -| TanStack AI | [`@telemetry-dev/tanstack-ai`](packages/tanstack-ai) | | -| LiteLLM | | [`telemetry-dev-litellm`](sdks/python-litellm) | -| MCP | [`@telemetry-dev/mcp`](packages/mcp) | | -| Eve | [`@telemetry-dev/eve`](packages/eve) | | -| Cursor | [`@telemetry-dev/cursor`](packages/cursor) | | -| OpenCode | [`@telemetry-dev/opencode`](packages/opencode) | | -| Oh My Pi | [`@telemetry-dev/omp`](packages/omp) | | -| Pi | [`@telemetry-dev/pi`](packages/pi) | | +| Integration | TypeScript | Python | +| ---------------------------- | -------------------------------------------------------- | -------------------------------------------------------- | +| Core SDK | [`@telemetry-dev/sdk`](packages/sdk) | [`telemetry-dev`](sdks/python) | +| Existing OpenTelemetry setup | [`@telemetry-dev/otel`](packages/otel) | [`telemetry-dev`](sdks/python) (`telemetry_dev.otel`) | +| OpenAI | [`@telemetry-dev/openai`](packages/openai) | [`telemetry-dev-openai`](sdks/python-openai) | +| Anthropic | [`@telemetry-dev/anthropic`](packages/anthropic) | [`telemetry-dev-anthropic`](sdks/python-anthropic) | +| Amazon Bedrock | [`@telemetry-dev/bedrock`](packages/bedrock) | [`telemetry-dev-bedrock`](sdks/python-bedrock) | +| Google GenAI | [`@telemetry-dev/google-genai`](packages/google-genai) | [`telemetry-dev-google-genai`](sdks/python-google-genai) | +| OpenRouter | [`@telemetry-dev/openrouter`](packages/openrouter) | [`telemetry-dev-openrouter`](sdks/python-openrouter) | +| Vercel AI SDK | [`@telemetry-dev/ai-sdk`](packages/ai) | | +| TanStack AI | [`@telemetry-dev/tanstack-ai`](packages/tanstack-ai) | | +| OpenAI Agents SDK | [`@telemetry-dev/openai-agents`](packages/openai-agents) | | +| LiteLLM | | [`telemetry-dev-litellm`](sdks/python-litellm) | +| MCP | [`@telemetry-dev/mcp`](packages/mcp) | | +| Eve | [`@telemetry-dev/eve`](packages/eve) | | +| Cursor | [`@telemetry-dev/cursor`](packages/cursor) | | +| OpenCode | [`@telemetry-dev/opencode`](packages/opencode) | | +| Oh My Pi | [`@telemetry-dev/omp`](packages/omp) | | +| Pi | [`@telemetry-dev/pi`](packages/pi) | | Some integrations need no package from this repository, because the tool exports OpenTelemetry itself or through a third-party library: Claude Code, Claude Platform, Claude SDK native spans, diff --git a/packages/openai-agents/LICENSE b/packages/openai-agents/LICENSE new file mode 100644 index 00000000..8592a259 --- /dev/null +++ b/packages/openai-agents/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 telemetry.dev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/openai-agents/README.md b/packages/openai-agents/README.md new file mode 100644 index 00000000..8205196f --- /dev/null +++ b/packages/openai-agents/README.md @@ -0,0 +1,108 @@ +# @telemetry-dev/openai-agents + +OpenAI Agents SDK integration for [telemetry.dev](https://telemetry.dev). It sends each agent run +to telemetry.dev as one trace: the agent and its turns, each model call with its token usage, each +tool call, and each command that a sandbox agent ran, with its exit code and the sandbox it ran in. + +## Install + +```sh +npm install @telemetry-dev/openai-agents @openai/agents +``` + +Requires `@openai/agents >=0.21.0 <1` and Node.js 22 or newer, the Agents SDK's minimum. + +## Environment + +| Variable | Required | Default | Notes | +| --------------------------- | -------- | --------------------------- | ----------------------------------------------------------------------- | +| `TELEMETRY_DEV_API_KEY` | yes | — | Ingest key (`td_live_…`). No key ⇒ the integration is a complete no-op. | +| `TELEMETRY_DEV_BASE_URL` | no | `https://api.telemetry.dev` | Trailing slashes are stripped. | +| `TELEMETRY_DEV_ENVIRONMENT` | no | `production` | Environment label on every trace. | +| `OTEL_SERVICE_NAME` | no | SDK default | Service name on every trace. | + +All four are also settable through the processor's options. + +## Usage + +Register the processor once, before the first run: + +```ts +import { addTraceProcessor, Agent, run } from "@openai/agents"; +import { telemetryDevProcessor } from "@telemetry-dev/openai-agents"; + +addTraceProcessor(telemetryDevProcessor({ serviceName: "support-agent" })); + +const agent = new Agent({ name: "Assistant", instructions: "Answer briefly." }); +await run(agent, "What is OpenTelemetry?"); +``` + +`addTraceProcessor` keeps the SDK's default exporter, so traces also reach the OpenAI Traces +dashboard. To send them only to telemetry.dev, replace the default exporter: + +```ts +import { setTraceProcessors } from "@openai/agents"; + +setTraceProcessors([telemetryDevProcessor({ serviceName: "support-agent" })]); +``` + +The processor takes the options of `init` from `@telemetry-dev/sdk`, such as `captureInput`, +`captureOutput`, and `mask`. Pass SDK options to the processor and don't also call `init()` +yourself: the first processor created in a process initializes the SDK and replaces a client that +`init()` made earlier. The options of later processors are ignored. + +A run that throws never ends its Agents trace, so its root span ends after a minute without new +spans, or at the next `forceFlush` or `shutdown` of the trace provider. Call +`getGlobalTraceProvider().forceFlush()` at the end of a request in short-lived runtimes. + +The Agents SDK shuts the trace provider down on `SIGTERM`, `SIGINT`, and unhandled rejections. The +processor then ends every open span so that nothing is lost if the process exits. A server that keeps +running to finish its requests still gets their later spans in the same traces, but a model call +that was in progress at that moment is sent without its token usage. + +## Sandbox agents + +Sandbox runs need no extra setup. Each `sandbox.*` span of the Agents SDK, such as +`sandbox.create_session` and `sandbox.exec`, becomes a span with `sandbox.operation` and, when the +run created its session, `sandbox.backend` (for example `e2b` or `modal`). The `exec_command` and `write_stdin` tool spans get +`process.exit.code` once the command has exited. A command that exits with a non-zero code is not +marked as an error, because many commands, such as `grep`, exit non-zero without failing. + +To also record the provider's own sandbox id, wrap the sandbox client: + +```ts +import { E2BSandboxClient } from "@openai/agents-extensions/sandbox/e2b"; +import { instrumentSandboxClient } from "@telemetry-dev/openai-agents"; + +const client = instrumentSandboxClient(new E2BSandboxClient()); +await run(agent, "Run the tests.", { sandbox: { client } }); +``` + +The session, command, and tool spans of that agent then carry `sandbox.id`, the id that the +provider uses in its own dashboard, logs, and webhooks. + +The backend and the id come from the span that creates or resumes the session. A session that the +app passes in with `sandbox: { session }`, or that a run resumed in the same process reuses while it +is still live, has no such span, so the spans of that run have neither. + +## Content and privacy + +- Model inputs and outputs, tool arguments and results, and the command and working directory of + `sandbox.exec` (`process.command_line`, `process.working_directory`) are content. They are sent + only when `captureInput` or `captureOutput` is on, and go through `mask`. +- Keep the Agents SDK's `traceIncludeSensitiveData` on, which is its default. With it off, the SDK + leaves model, token usage, and tool output off its spans, so telemetry.dev cannot price model + calls or read exit codes. Use `captureInput` and `captureOutput` to keep content out of + telemetry.dev instead. With `addTraceProcessor`, the OpenAI exporter still receives content; use + `setTraceProcessors` to stop that. +- With capture off, telemetry.dev still receives span names and timing, model names, token usage, + agent, tool, handoff, and guardrail names, the MCP server of a tool call, exit codes, sandbox + backends and ids, error types, the trace's `groupId` as the session id, and the string, number, + and boolean values of trace metadata and of the data of custom spans. Metadata does not go + through `mask`. + +## Double counting + +Each model call already gets a `chat` span from this processor. Do not also pass a client wrapped +with `wrapOpenAI` from `@telemetry-dev/openai` to `setDefaultOpenAIClient`: each model call would +then be counted twice. diff --git a/packages/openai-agents/package.json b/packages/openai-agents/package.json new file mode 100644 index 00000000..6a375c00 --- /dev/null +++ b/packages/openai-agents/package.json @@ -0,0 +1,73 @@ +{ + "name": "@telemetry-dev/openai-agents", + "version": "0.1.0", + "description": "OpenAI Agents SDK telemetry integration for telemetry.dev: agent runs, model calls, tool calls, and sandbox commands as OpenTelemetry traces.", + "keywords": [ + "agents", + "genai", + "llm", + "observability", + "openai", + "openai-agents", + "opentelemetry", + "sandbox", + "telemetry", + "tracing" + ], + "homepage": "https://telemetry.dev", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/telemetry-dev/telemetry.git", + "directory": "packages/openai-agents" + }, + "files": [ + "dist", + "src" + ], + "type": "module", + "exports": { + ".": { + "bun": "./src/index.ts", + "types": "./src/index.ts", + "development": "./src/index.ts", + "default": "./dist/index.mjs" + }, + "./package.json": "./package.json" + }, + "publishConfig": { + "access": "public", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs", + "default": "./dist/index.mjs" + }, + "./package.json": "./package.json" + } + }, + "scripts": { + "build": "pnpm exec vp pack", + "dev": "pnpm exec vp pack --watch", + "test": "vp test", + "check": "vp check", + "prepublishOnly": "vp run build" + }, + "dependencies": { + "@telemetry-dev/sdk": "workspace:^" + }, + "devDependencies": { + "@openai/agents": "0.21.0", + "@opentelemetry/sdk-trace-base": "catalog:", + "@types/node": "^25.5.0", + "@typescript/native-preview": "7.0.0-dev.20260328.1", + "openai": "catalog:", + "typescript": "^6.0.2", + "vite-plus": "0.1.20", + "vitest": "catalog:", + "zod": "catalog:" + }, + "peerDependencies": { + "@openai/agents": ">=0.21.0 <1" + } +} diff --git a/packages/openai-agents/src/config.ts b/packages/openai-agents/src/config.ts new file mode 100644 index 00000000..c401ec1b --- /dev/null +++ b/packages/openai-agents/src/config.ts @@ -0,0 +1,28 @@ +import { type ClientOverrides, init, shutdown, type TelemetryOptions } from "@telemetry-dev/sdk"; + +/** Options accepted by `telemetryDevProcessor`; see `TelemetryOptions` of `@telemetry-dev/sdk`. */ +export type TelemetryDevOpenAIAgentsOptions = Omit; + +export type { ClientOverrides }; + +let initialized = false; + +/** + * Initializes the telemetry.dev SDK once per process for this integration; whichever processor is + * created first wins. The processor builds spans with the SDK's own tracer, so it does not need + * the global OpenTelemetry provider. + */ +export function ensureInit( + options: TelemetryDevOpenAIAgentsOptions = {}, + overrides?: ClientOverrides, +): void { + if (initialized) return; + initialized = true; + init({ ...options, sdkName: "@telemetry-dev/openai-agents" }, overrides); +} + +/** Test seam for unit tests that need a fresh SDK singleton. */ +export async function resetForTesting(): Promise { + initialized = false; + await shutdown(); +} diff --git a/packages/openai-agents/src/index.ts b/packages/openai-agents/src/index.ts new file mode 100644 index 00000000..8165e977 --- /dev/null +++ b/packages/openai-agents/src/index.ts @@ -0,0 +1,5 @@ +export type { ClientOverrides, TelemetryDevOpenAIAgentsOptions } from "./config.ts"; + +export { telemetryDevProcessor } from "./processor.ts"; + +export { instrumentSandboxClient } from "./sandbox.ts"; diff --git a/packages/openai-agents/src/processor.ts b/packages/openai-agents/src/processor.ts new file mode 100644 index 00000000..06af8920 --- /dev/null +++ b/packages/openai-agents/src/processor.ts @@ -0,0 +1,599 @@ +import type { Span, SpanData, Trace, TracingProcessor } from "@openai/agents"; +import { + captureEnabled, + flush, + type SpanFields, + type SpanHandle, + startSpan, + type StartSpanOptions, + type TokenUsage, +} from "@telemetry-dev/sdk"; + +import { + type ClientOverrides, + ensureInit, + type TelemetryDevOpenAIAgentsOptions, +} from "./config.ts"; +import { SANDBOX_ID_KEY } from "./sandbox.ts"; + +type AgentsSpan = Span; + +type Attrs = NonNullable; + +// Bounds the processor's memory when a host never ends its spans; past the cap new spans are +// dropped, not exported. +const MAX_OPEN_SPANS = 10_000; + +// How long a trace with no open spans waits for `onTraceEnd` before its root ends anyway, and how +// long a finished trace is kept afterwards so that late spans still join it. The Agents SDK ends a +// trace only when its run returns; a run that throws never ends it. +const IDLE_TRACE_MS = 60_000; + +// Bounds the finished traces kept for late spans when runs finish faster than they expire. +const MAX_TRACES = 10_000; + +// `formatExecResponse` in agents-core writes this line in the header of a shell tool's output, +// above `Output:`, once the command has exited. A command still running gets a session id line +// instead, and the command's own output below the header can contain any text. +const EXIT_CODE_LINE = /^Process exited with code (-?\d+)$/m; +const OUTPUT_MARKER = "\nOutput:"; + +const SANDBOX_COMMAND_TOOLS = new Set(["exec_command", "write_stdin"]); + +const SANDBOX_SPAN_PREFIX = "sandbox."; + +interface SandboxRef { + backend?: string; + id?: string; +} + +/** Sandbox of each agent in one run, keyed by agent name. */ +type RunSandboxes = Map; + +interface TraceState { + root: SpanHandle; + open: number; + /** When `onTraceEnd` arrived. The root ends once this is set and no span is open. */ + endTime?: Date; + lastSpanEnd?: Date; + /** The root has ended. The state stays until it expires, so a late span joins the root. */ + finished: boolean; + timer?: ReturnType; +} + +interface SpanState { + handle: SpanHandle; + /** The trace state this span was counted in, if any. */ + trace?: TraceState; + /** `shutdown` ended the handle; the SDK's own end of the span only uncounts it. */ + ended?: boolean; + /** Name of the agent this span runs under, for looking up its sandbox. */ + agent?: string; + /** Sandboxes of the run this span belongs to; shared by every span of the run. */ + sandboxes?: RunSandboxes; +} + +type JsonRecord = Record; + +const asRecord = (value: unknown): JsonRecord | undefined => + value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonRecord) + : undefined; + +const asString = (value: unknown): string | undefined => + typeof value === "string" && value.length > 0 ? value : undefined; + +const asNumber = (value: unknown): number | undefined => + typeof value === "number" && Number.isFinite(value) ? value : undefined; + +const toTime = (iso: string | null): Date | undefined => (iso ? new Date(iso) : undefined); + +const compact = (usage: TokenUsage): TokenUsage | undefined => + Object.values(usage).some((v) => v !== undefined) ? usage : undefined; + +/** + * Usage of a Responses API response (`input_tokens_details`), or of a `generation` span, whose + * adapters put token details under `details`. + */ +function usageOf(value: unknown): TokenUsage | undefined { + const usage = asRecord(value); + + if (!usage) return undefined; + const details = asRecord(usage.details); + const inputDetails = asRecord(usage.input_tokens_details) ?? details; + const outputDetails = asRecord(usage.output_tokens_details) ?? details; + + return compact({ + inputTokens: asNumber(usage.input_tokens), + outputTokens: asNumber(usage.output_tokens), + totalTokens: asNumber(usage.total_tokens), + cacheReadInputTokens: asNumber(inputDetails?.cached_tokens), + reasoningOutputTokens: asNumber(outputDetails?.reasoning_tokens), + }); +} + +/** Usage of a Chat Completions response, which a successful `generation` span keeps in its output. */ +function chatCompletionUsageOf(value: unknown): TokenUsage | undefined { + const usage = asRecord(value); + + if (!usage) return undefined; + + return compact({ + inputTokens: asNumber(usage.prompt_tokens), + outputTokens: asNumber(usage.completion_tokens), + totalTokens: asNumber(usage.total_tokens), + cacheReadInputTokens: asNumber(asRecord(usage.prompt_tokens_details)?.cached_tokens), + reasoningOutputTokens: asNumber(asRecord(usage.completion_tokens_details)?.reasoning_tokens), + }); +} + +function primitiveMetadata(value: unknown): Record | undefined { + const record = asRecord(value); + + if (!record) return undefined; + const out: Record = {}; + + for (const [key, v] of Object.entries(record)) { + if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") out[key] = v; + } + + return Object.keys(out).length > 0 ? out : undefined; +} + +function agentOf(data: SpanData): string | undefined { + switch (data.type) { + case "agent": + return data.name; + case "turn": + return data.agent_name; + case "custom": + return asString(data.data.agent_name); + default: + return undefined; + } +} + +/** Name and fields known when the span starts. */ +function startOf(data: SpanData): { name: string; options: StartSpanOptions } { + switch (data.type) { + case "agent": + return { name: data.name, options: { type: "agent", agentName: data.name } }; + case "task": + return { name: `task ${data.name}`, options: {} }; + case "turn": + return { + name: `turn ${data.turn}`, + options: { attributes: { "openai_agents.turn": data.turn } }, + }; + case "response": + return { name: "chat", options: { type: "generation", provider: "openai" } }; + case "generation": + return { + name: data.model ? `chat ${data.model}` : "chat", + options: { type: "generation", model: data.model }, + }; + case "function": + return { name: data.name, options: { type: "tool", toolName: data.name } }; + case "handoff": + return { name: "handoff", options: {} }; + case "guardrail": + return { + name: `guardrail ${data.name}`, + options: { attributes: { "openai_agents.guardrail.name": data.name } }, + }; + case "mcp_tools": + return { name: data.server ? `mcp_tools ${data.server}` : "mcp_tools", options: {} }; + case "custom": + return { name: data.name, options: {} }; + default: + return { name: data.type, options: {} }; + } +} + +/** Fields known when the span ends, apart from sandbox attributes. */ +function endOf(data: SpanData): SpanFields { + switch (data.type) { + // The SDK fills the tool list once the agent's tools are resolved, after the span starts. + case "agent": + return { + attributes: { + ...(data.tools?.length ? { "openai_agents.agent.tools": data.tools } : {}), + ...(data.handoffs?.length ? { "openai_agents.agent.handoffs": data.handoffs } : {}), + ...(data.output_type ? { "openai_agents.agent.output_type": data.output_type } : {}), + }, + }; + case "response": { + const response = asRecord(data._response); + const model = asString(response?.model); + + return { + ...(model ? { name: `chat ${model}`, model } : {}), + responseModel: model, + responseId: data.response_id ?? asString(response?.id), + usage: usageOf(response?.usage), + input: data._input, + output: response?.output as SpanFields["output"], + }; + } + + // The Chat Completions model sets the model after the span starts, and on success leaves + // `usage` empty: the usage is only in the completion it stores as output. + case "generation": { + const completion = asRecord(data.output?.[0]); + const model = asString(data.model); + + return { + ...(model ? { name: `chat ${model}`, model } : {}), + responseModel: asString(completion?.model), + responseId: asString(completion?.id), + usage: usageOf(data.usage) ?? chatCompletionUsageOf(completion?.usage), + input: data.input as SpanFields["input"], + output: data.output as SpanFields["output"], + }; + } + + case "function": { + // `mcp_data` is typed as a string but holds `{ server }` at runtime. + const mcpServer = asString(asRecord(data.mcp_data)?.server) ?? asString(data.mcp_data); + + return { + input: data.input, + output: data.output, + ...(mcpServer ? { attributes: { "openai_agents.mcp.server": mcpServer } } : {}), + }; + } + + case "handoff": + return { + name: + data.from_agent && data.to_agent + ? `handoff ${data.from_agent} -> ${data.to_agent}` + : undefined, + attributes: { + ...(data.from_agent ? { "openai_agents.handoff.from": data.from_agent } : {}), + ...(data.to_agent ? { "openai_agents.handoff.to": data.to_agent } : {}), + }, + }; + case "guardrail": + return { attributes: { "openai_agents.guardrail.triggered": data.triggered } }; + case "mcp_tools": + return data.result + ? { attributes: { "openai_agents.mcp.tool_count": data.result.length } } + : {}; + case "custom": + return data.name.startsWith(SANDBOX_SPAN_PREFIX) + ? {} + : { metadata: primitiveMetadata(data.data) }; + default: + return {}; + } +} + +function errorOf(span: AgentsSpan): Error | undefined { + const error = span.error; + + if (!error) return undefined; + const sandboxError = + span.spanData.type === "custom" ? asRecord(span.spanData.data.error) : undefined; + const result = new Error(asString(sandboxError?.message) ?? error.message); + result.name = asString(sandboxError?.name) ?? "Error"; + + return result; +} + +function exitCodeOf(output: string | undefined): number | undefined { + if (!output) return undefined; + const marker = output.indexOf(OUTPUT_MARKER); + const header = marker < 0 ? output : output.slice(0, marker); + const code = EXIT_CODE_LINE.exec(header)?.[1]; + + return code === undefined ? undefined : Number(code); +} + +function mergeFields(...parts: SpanFields[]): SpanFields { + const merged: SpanFields = {}; + + for (const part of parts) { + const { attributes, ...rest } = part; + Object.assign(merged, rest); + + if (attributes) merged.attributes = { ...merged.attributes, ...attributes }; + } + + return merged; +} + +const sandboxAttributes = (ref: SandboxRef | undefined): Attrs => ({ + ...(ref?.backend ? { "sandbox.backend": ref.backend } : {}), + ...(ref?.id ? { "sandbox.id": ref.id } : {}), +}); + +/** + * Sends OpenAI Agents SDK traces to telemetry.dev. Register it with `addTraceProcessor`, which + * keeps the SDK's default OpenAI exporter, or `setTraceProcessors([...])`, which replaces it. + * + * Each Agents trace gets one root span, and its spans keep their parents. Model calls are `chat` + * spans and the only spans with token usage; task, turn, and agent spans repeat the totals of their + * calls and carry none. Sandbox spans keep their `sandbox.*` names and get `sandbox.backend`, + * `sandbox.id`, and `sandbox.operation`. The `exec_command` and `write_stdin` tool spans get + * `process.exit.code` once the command has exited. Command text and working directory are input + * content: they are sent only when `captureInput` is on, and go through the mask. + * + * The root span ends when the trace has ended and every one of its spans has ended, since other + * processors can delay span ends past the trace end. A trace of a run that threw ends after + * `IDLE_TRACE_MS` without open spans, or at the next `forceFlush` or `shutdown`. + */ +export function telemetryDevProcessor( + options: TelemetryDevOpenAIAgentsOptions = {}, + overrides?: ClientOverrides, +): TracingProcessor { + const onError = options.onError; + + const report = (cause: unknown): void => { + try { + onError?.(cause instanceof Error ? cause : new Error(String(cause))); + } catch { + // A failing onError must not fail an agent run. + } + }; + + try { + ensureInit(options, overrides); + } catch (error) { + report(error); + } + + const traces = new Map(); + const spans = new Map(); + let reportedCap = false; + + function finish(state: TraceState): void { + if (state.finished) return; + state.finished = true; + state.root.end({ endTime: state.endTime ?? state.lastSpanEnd }); + } + + function remove(traceId: string, state: TraceState): void { + clearTimeout(state.timer); + + if (traces.get(traceId) === state) traces.delete(traceId); + } + + /** Ends the root or schedules its end or expiry, from the trace's open span count. */ + function settle(traceId: string, state: TraceState): void { + clearTimeout(state.timer); + state.timer = undefined; + + if (state.open > 0) return; + + if (state.endTime) finish(state); + + // An unfinished trace waits for `onTraceEnd`; a finished one waits for late spans. + state.timer = setTimeout(() => { + try { + finish(state); + remove(traceId, state); + } catch (error) { + report(error); + } + }, IDLE_TRACE_MS); + state.timer.unref?.(); + } + + function uncount( + traceId: string, + state: TraceState | undefined, + endTime: Date | undefined, + ): void { + if (!state) return; + state.open--; + + if (endTime && (!state.lastSpanEnd || endTime > state.lastSpanEnd)) state.lastSpanEnd = endTime; + settle(traceId, state); + } + + function evictFinished(): void { + for (const [traceId, state] of traces) { + if (traces.size < MAX_TRACES) return; + + if (state.finished && state.open === 0) remove(traceId, state); + } + } + + function sandboxSpanEnd(span: AgentsSpan, state: SpanState): SpanFields { + const data = span.spanData; + + if (data.type !== "custom" || !data.name.startsWith(SANDBOX_SPAN_PREFIX)) return {}; + const backend = asString(data.data.backend_id); + const id = asString(data.data[SANDBOX_ID_KEY]); + const own: SandboxRef = { ...(backend ? { backend } : {}), ...(id ? { id } : {}) }; + + if (state.agent && state.sandboxes && (backend || id)) { + state.sandboxes.set(state.agent, { ...state.sandboxes.get(state.agent), ...own }); + } + + const attributes: Attrs = { + "sandbox.operation": data.name.slice(SANDBOX_SPAN_PREFIX.length), + ...sandboxAttributes({ ...sandboxFor(state), ...own }), + }; + + if (data.name === "sandbox.exec" && captureEnabled("input")) { + const command = asString(data.data.cmd); + const workdir = asString(data.data.workdir); + const commandLine = + command && state.handle.serializeContent?.(command, "process.command_line"); + const directory = + workdir && state.handle.serializeContent?.(workdir, "process.working_directory"); + + if (commandLine) attributes["process.command_line"] = commandLine; + + if (directory) attributes["process.working_directory"] = directory; + } + + return { attributes }; + } + + function commandToolEnd(span: AgentsSpan, state: SpanState): SpanFields { + const data = span.spanData; + + if (data.type !== "function" || !SANDBOX_COMMAND_TOOLS.has(data.name)) return {}; + const exitCode = exitCodeOf(data.output); + const attributes: Attrs = sandboxAttributes(sandboxFor(state)); + + // A non-zero exit is recorded, not raised: `grep` exits 1 when nothing matches. + if (exitCode !== undefined) attributes["process.exit.code"] = exitCode; + + return Object.keys(attributes).length > 0 ? { attributes } : {}; + } + + const sandboxFor = (state: SpanState): SandboxRef | undefined => + state.agent ? state.sandboxes?.get(state.agent) : undefined; + + return { + async onTraceStart(trace: Trace) { + try { + const existing = traces.get(trace.traceId); + + // A trace id seen again, such as a resumed run's or a fixed `traceId`, keeps its root while + // the processor still holds it, even when that root has already ended. + if (existing) { + existing.endTime = undefined; + settle(trace.traceId, existing); + + return; + } + + evictFinished(); + + const attributes: Attrs = { "openai_agents.trace_id": trace.traceId }; + + if (trace.groupId) attributes["gen_ai.conversation.id"] = trace.groupId; + const root = startSpan(trace.name, { + attributes, + metadata: primitiveMetadata(trace.metadata), + }); + const state: TraceState = { root, open: 0, finished: false }; + traces.set(trace.traceId, state); + settle(trace.traceId, state); + } catch (error) { + report(error); + } + }, + + async onTraceEnd(trace: Trace) { + try { + const state = traces.get(trace.traceId); + + if (!state) return; + state.endTime = new Date(); + settle(trace.traceId, state); + } catch (error) { + report(error); + } + }, + + async onSpanStart(span: AgentsSpan) { + try { + if (spans.size >= MAX_OPEN_SPANS) { + if (!reportedCap) { + reportedCap = true; + report(new Error(`more than ${MAX_OPEN_SPANS} open Agents spans; dropping new spans`)); + } + + return; + } + + const data = span.spanData; + const parent = span.parentId ? spans.get(span.parentId) : undefined; + const traceState = traces.get(span.traceId); + const { name, options } = startOf(data); + const handle = startSpan(name, { + ...options, + parent: parent?.handle ?? traceState?.root, + startTime: toTime(span.startedAt), + }); + + // A task span is one run; an agent span without one is a run without task spans. + const sandboxes = + data.type === "task" || (data.type === "agent" && !parent?.sandboxes) + ? new Map() + : parent?.sandboxes; + + spans.set(span.spanId, { + handle, + trace: traceState, + agent: agentOf(data) ?? parent?.agent, + sandboxes, + }); + + if (traceState) { + traceState.open++; + settle(span.traceId, traceState); + } + } catch (error) { + report(error); + } + }, + + async onSpanEnd(span: AgentsSpan) { + try { + const state = spans.get(span.spanId); + + if (!state) return; + spans.delete(span.spanId); + + if (state.ended) { + uncount(span.traceId, state.trace, undefined); + + return; + } + + const fields = mergeFields( + endOf(span.spanData), + sandboxSpanEnd(span, state), + commandToolEnd(span, state), + ); + const error = errorOf(span); + const endTime = toTime(span.endedAt); + + state.handle.end({ ...fields, ...(error ? { error } : {}), endTime }); + uncount(span.traceId, state.trace, endTime ?? new Date()); + } catch (error) { + report(error); + } + }, + + // Ends the roots of traces whose run threw. A trace that has not ended a span yet may be a + // run that is still preparing its first span, so it keeps waiting. + async forceFlush() { + try { + for (const state of traces.values()) { + if (state.open === 0 && (state.endTime || state.lastSpanEnd)) finish(state); + } + } catch (error) { + report(error); + } + + await flush(); + }, + + // The SDK also calls this on SIGTERM, SIGINT, and unhandled rejections, after which a server + // that drains requests keeps running. So spans and traces stay known: later spans join their + // parents, and the SDK's own ends of the spans ended here only uncount them. + async shutdown() { + try { + for (const state of spans.values()) { + if (state.ended) continue; + state.ended = true; + state.handle.end(); + } + + for (const state of traces.values()) finish(state); + } catch (error) { + report(error); + } + + await flush(); + }, + }; +} diff --git a/packages/openai-agents/src/sandbox.ts b/packages/openai-agents/src/sandbox.ts new file mode 100644 index 00000000..29320527 --- /dev/null +++ b/packages/openai-agents/src/sandbox.ts @@ -0,0 +1,47 @@ +import { getCurrentSpan } from "@openai/agents"; +import type { SandboxClient } from "@openai/agents/sandbox"; + +/** + * Key under which `instrumentSandboxClient` writes the provider's sandbox id into the data of the + * `sandbox.create_session` or `sandbox.resume_session` span. The Agents SDK records only its + * backend id there; the provider id lives in the session state. + */ +export const SANDBOX_ID_KEY = "sandbox_id"; + +type SessionFactory = (...args: never[]) => Promise<{ state?: { sandboxId?: unknown } }>; + +/** + * Records the provider's sandbox id (for example E2B's `sandboxId`) on the Agents SDK span that + * creates or resumes the session, so telemetry.dev can join the provider's own sandbox events to + * the trace. Patches `create` and `resume` on this client instance and returns the same instance. + */ +export function instrumentSandboxClient(client: T): T { + // `create` and `resume` are generic over each client's options and state. The wrapper only + // forwards its arguments and reads `state.sandboxId`, so one loose signature is enough. + const target = client as unknown as Record<"create" | "resume", SessionFactory | undefined>; + + for (const method of ["create", "resume"] as const) { + const original = target[method]; + + if (original === undefined) continue; + + target[method] = async (...args) => { + const session = await original.apply(client, args); + + try { + const sandboxId = session?.state?.sandboxId; + const span = getCurrentSpan(); + + if (typeof sandboxId === "string" && span?.spanData.type === "custom") { + span.spanData.data[SANDBOX_ID_KEY] = sandboxId; + } + } catch { + // Telemetry must never fail sandbox creation. + } + + return session; + }; + } + + return client; +} diff --git a/packages/openai-agents/tests/openai-agents.test.ts b/packages/openai-agents/tests/openai-agents.test.ts new file mode 100644 index 00000000..811bf6d6 --- /dev/null +++ b/packages/openai-agents/tests/openai-agents.test.ts @@ -0,0 +1,602 @@ +import { + type Model, + OpenAIChatCompletionsModel, + OpenAIResponsesModel, + Runner, + Span, + setTraceProcessors, + setTracingDisabled, + Trace, + type TracingProcessor, + withCustomSpan, + withTrace, +} from "@openai/agents"; +import { SandboxAgent, shell } from "@openai/agents/sandbox"; +import { UnixLocalSandboxClient } from "@openai/agents/sandbox/local"; +import { InMemorySpanExporter, type ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { flush } from "@telemetry-dev/sdk"; +import OpenAI from "openai"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { resetForTesting } from "../src/config.ts"; +import { instrumentSandboxClient, telemetryDevProcessor } from "../src/index.ts"; + +type Json = Record; + +let exporter: InMemorySpanExporter; + +type ProcessorOptions = Parameters[0]; + +function makeProcessor( + options: ProcessorOptions = {}, + before: TracingProcessor[] = [], +): TracingProcessor { + exporter = new InMemorySpanExporter(); + const processor = telemetryDevProcessor( + { + apiKey: "td_live_test", + environment: "test", + exportMode: "immediate", + logLevel: "silent", + fetch: async () => new Response(null, { status: 200 }), + ...options, + }, + { spanExporter: exporter }, + ); + setTraceProcessors([...before, processor]); + + return processor; +} + +beforeEach(() => { + // The Agents SDK turns tracing off when NODE_ENV is "test". + setTracingDisabled(false); +}); + +afterEach(async () => { + setTraceProcessors([]); + await resetForTesting(); +}); + +function functionCall(command: string, callId: string): Json { + return { + type: "function_call", + id: `fc_${callId}`, + call_id: callId, + name: "exec_command", + arguments: JSON.stringify({ cmd: command, login: false }), + status: "completed", + }; +} + +function message(text: string): Json { + return { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text, annotations: [] }], + }; +} + +function response(id: string, output: Json[], usage: Json): Json { + return { + id, + object: "response", + created_at: 0, + model: "gpt-5-mini", + status: "completed", + output, + usage, + }; +} + +/** An OpenAI client whose HTTP calls return `bodies` in order. */ +function scriptedClient(bodies: Json[], status = 200): OpenAI { + const queue = [...bodies]; + + return new OpenAI({ + apiKey: "sk-test", + maxRetries: 0, + fetch: async () => { + const body = queue.shift(); + + if (!body) throw new Error("no scripted response left"); + + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }); +} + +/** A Responses API model: the model asks for `command`, then answers "done". */ +function responsesModel(command: string): Model { + return new OpenAIResponsesModel( + scriptedClient([ + response("resp_1", [functionCall(command, "call_1")], { + input_tokens: 100, + output_tokens: 20, + total_tokens: 120, + input_tokens_details: { cached_tokens: 40 }, + output_tokens_details: { reasoning_tokens: 5 }, + }), + response("resp_2", [message("done")], { + input_tokens: 50, + output_tokens: 7, + total_tokens: 57, + }), + ]), + "gpt-5-mini", + ); +} + +function completion(id: string, message: Json, usage: Json): Json { + return { + id, + object: "chat.completion", + created: 0, + model: "gpt-4.1-mini-2025-04-14", + choices: [{ index: 0, message, finish_reason: "stop", logprobs: null }], + usage, + }; +} + +/** A Chat Completions model with the same script as `responsesModel`. */ +function chatCompletionsModel(command: string): Model { + return new OpenAIChatCompletionsModel( + scriptedClient([ + completion( + "chatcmpl_1", + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "exec_command", arguments: JSON.stringify({ cmd: command }) }, + }, + ], + }, + { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 40 }, + completion_tokens_details: { reasoning_tokens: 5 }, + }, + ), + completion( + "chatcmpl_2", + { role: "assistant", content: "done" }, + { prompt_tokens: 50, completion_tokens: 7, total_tokens: 57 }, + ), + ]), + "gpt-4.1-mini", + ); +} + +/** One run of a sandbox agent with `model`, by default one that runs `command`. */ +async function runCommand( + command: string, + options: { + sandboxClient?: UnixLocalSandboxClient; + workflowName?: string; + model?: Model; + } = {}, +): Promise { + const agent = new SandboxAgent({ + name: "coder", + instructions: "Run the command.", + model: options.model ?? responsesModel(command), + capabilities: [shell()], + }); + + await new Runner({ workflowName: options.workflowName ?? "workflow" }).run(agent, "go", { + sandbox: { client: options.sandboxClient ?? new UnixLocalSandboxClient() }, + }); +} + +async function exported(): Promise { + await flush(); + + return exporter.getFinishedSpans(); +} + +const parentOf = (span: ReadableSpan): string | undefined => span.parentSpanContext?.spanId; + +function only(spans: ReadableSpan[], predicate: (span: ReadableSpan) => boolean): ReadableSpan { + const matches = spans.filter(predicate); + expect(matches).toHaveLength(1); + + return matches[0]!; +} + +const named = (name: string) => (span: ReadableSpan) => span.name === name; + +test("a sandbox command becomes a tool span with its exit code, nested under its agent", async () => { + makeProcessor(); + await runCommand("sh -c 'exit 3'"); + const spans = await exported(); + + const root = only(spans, named("workflow")); + const tool = only(spans, named("exec_command")); + const exec = only(spans, named("sandbox.exec")); + const turn = spans.find((span) => span.spanContext().spanId === parentOf(tool))!; + const agent = spans.find((span) => span.spanContext().spanId === parentOf(turn))!; + + expect(new Set(spans.map((span) => span.spanContext().traceId)).size).toBe(1); + expect(tool.attributes["gen_ai.operation.name"]).toBe("execute_tool"); + expect(tool.attributes["process.exit.code"]).toBe(3); + // A non-zero exit is not a failed tool call. + expect(tool.status.code).not.toBe(2); + expect(parentOf(exec)).toBe(tool.spanContext().spanId); + expect(exec.attributes["sandbox.operation"]).toBe("exec"); + expect(exec.attributes["sandbox.backend"]).toBe("unix_local"); + expect(exec.attributes["process.command_line"]).toBe("sh -c 'exit 3'"); + expect(turn.name).toMatch(/^turn /); + expect(agent.attributes["gen_ai.operation.name"]).toBe("invoke_agent"); + expect(agent.attributes["gen_ai.agent.name"]).toBe("coder"); + expect(spans.every((span) => span === root || parentOf(span) !== undefined)).toBe(true); +}); + +test("a command that exits 0 records exit code 0", async () => { + makeProcessor(); + await runCommand("true"); + + const tool = only(await exported(), named("exec_command")); + expect(tool.attributes["process.exit.code"]).toBe(0); +}); + +test("only model-call spans carry token usage", async () => { + makeProcessor(); + await runCommand("true"); + const spans = await exported(); + + const chats = spans.filter((span) => span.attributes["gen_ai.operation.name"] === "chat"); + expect(chats.map((span) => span.name)).toEqual(["chat gpt-5-mini", "chat gpt-5-mini"]); + const sum = (key: string) => + chats.reduce((total, span) => total + Number(span.attributes[key] ?? 0), 0); + expect(sum("gen_ai.usage.input_tokens")).toBe(150); + expect(sum("gen_ai.usage.output_tokens")).toBe(27); + expect(sum("gen_ai.usage.cache_read.input_tokens")).toBe(40); + expect(sum("gen_ai.usage.reasoning.output_tokens")).toBe(5); + expect(new Set(chats.map((span) => span.attributes["gen_ai.response.id"]))).toEqual( + new Set(["resp_1", "resp_2"]), + ); + + const withUsage = spans.filter((span) => + Object.keys(span.attributes).some((key) => key.startsWith("gen_ai.usage.")), + ); + expect(withUsage).toEqual(chats); +}); + +test("with captureInput off, the command is not on the sandbox or tool spans", async () => { + makeProcessor({ captureInput: false }); + await runCommand("echo hello"); + const spans = await exported(); + + expect(only(spans, named("sandbox.exec")).attributes["process.command_line"]).toBeUndefined(); + expect( + only(spans, named("exec_command")).attributes["gen_ai.tool.call.arguments"], + ).toBeUndefined(); +}); + +test("with capture off, no attribute contains the command or its output", async () => { + // The command is also the model's output (its function call) and the tool's output (the echo). + makeProcessor({ captureInput: false, captureOutput: false }); + const marker = "MARKER_4f2a9c"; + await runCommand(`echo ${marker}`); + const spans = await exported(); + + const leaked = spans.filter((span) => + Object.values(span.attributes).some((value) => JSON.stringify(value).includes(marker)), + ); + expect(leaked.map((span) => span.name)).toEqual([]); + expect(only(spans, named("exec_command")).attributes["process.exit.code"]).toBe(0); +}); + +/** A local sandbox client whose sessions carry `sandboxId`, as a hosted backend such as E2B's do. */ +function clientWithId(sandboxId: string): UnixLocalSandboxClient { + const client = new UnixLocalSandboxClient(); + const create = client.create.bind(client); + client.create = (async (...args: Parameters) => { + const session = await create(...args); + session.state.sandboxId = sandboxId; + + return session; + }) as typeof client.create; + + return instrumentSandboxClient(client); +} + +test("the provider's sandbox id reaches the session, command, and tool spans", async () => { + makeProcessor(); + await runCommand("true", { sandboxClient: clientWithId("sbx_test") }); + const spans = await exported(); + + for (const name of ["sandbox.create_session", "sandbox.exec", "exec_command"]) { + expect(only(spans, named(name)).attributes["sandbox.id"], name).toBe("sbx_test"); + } +}); + +test("two concurrent runs produce two traces whose spans do not cross", async () => { + makeProcessor(); + await Promise.all([ + runCommand("echo run_a", { workflowName: "run a" }), + runCommand("echo run_b", { workflowName: "run b" }), + ]); + const spans = await exported(); + + for (const run of ["a", "b"]) { + const root = only(spans, named(`run ${run}`)); + const traceId = root.spanContext().traceId; + const commands = spans + .filter((span) => span.spanContext().traceId === traceId && span.name === "sandbox.exec") + .map((span) => span.attributes["process.command_line"]); + expect(commands).toEqual([`echo run_${run}`]); + } +}); + +test("a span whose parent is unknown nests under its trace root", async () => { + const processor = makeProcessor(); + const trace = new Trace({ traceId: "trace_0123456789abcdef0123456789abcdef", name: "manual" }); + await processor.onTraceStart(trace); + const span = new Span( + { + traceId: trace.traceId, + parentId: "span_missing", + data: { type: "custom", name: "step", data: { attempt: 2 } }, + }, + processor, + ); + span.start(); + span.end(); + await processor.onTraceEnd(trace); + const spans = await exported(); + + const root = only(spans, named("manual")); + const step = only(spans, named("step")); + expect(parentOf(step)).toBe(root.spanContext().spanId); + expect(step.attributes["td.metadata.attempt"]).toBe("2"); +}); + +test("two runs under one trace keep their own sandbox ids", async () => { + makeProcessor(); + // Both agents are named "coder", so a lookup by name alone would mix the sandboxes up. + await withTrace("batch", () => + Promise.all([ + runCommand("echo a", { sandboxClient: clientWithId("sbx_a") }), + runCommand("echo b", { sandboxClient: clientWithId("sbx_b") }), + ]), + ); + const spans = await exported(); + + const execs = spans.filter(named("sandbox.exec")); + expect( + Object.fromEntries( + execs.map((span) => [span.attributes["process.command_line"], span.attributes["sandbox.id"]]), + ), + ).toEqual({ "echo a": "sbx_a", "echo b": "sbx_b" }); + + for (const exec of execs) { + const tool = spans.find((span) => span.spanContext().spanId === parentOf(exec))!; + expect(tool.attributes["sandbox.id"]).toBe(exec.attributes["sandbox.id"]); + } +}); + +test("instrumentSandboxClient records the id of a resumed session", async () => { + makeProcessor(); + const client = new UnixLocalSandboxClient(); + client.resume = (async () => ({ state: { sandboxId: "sbx_resumed" } })) as never; + instrumentSandboxClient(client); + + await withTrace("resume", () => + withCustomSpan(async () => await client.resume!({} as never), { + data: { + name: "sandbox.resume_session", + data: { agent_name: "coder", backend_id: "unix_local" }, + }, + }), + ); + const span = only(await exported(), named("sandbox.resume_session")); + + expect(span.attributes["sandbox.id"]).toBe("sbx_resumed"); + expect(span.attributes["sandbox.backend"]).toBe("unix_local"); +}); + +test("a mask applies to the command before it is sent", async () => { + makeProcessor({ + mask: (value, { key }) => (key === "process.command_line" ? "[masked]" : value), + }); + await runCommand("echo secret"); + + const exec = only(await exported(), named("sandbox.exec")); + expect(exec.attributes["process.command_line"]).toBe("[masked]"); +}); + +test("Chat Completions model calls carry their model and usage", async () => { + makeProcessor(); + await runCommand("true", { model: chatCompletionsModel("true") }); + const spans = await exported(); + + const chats = spans.filter((span) => span.attributes["gen_ai.operation.name"] === "chat"); + expect(chats.map((span) => span.name)).toEqual(["chat gpt-4.1-mini", "chat gpt-4.1-mini"]); + const first = chats.find((span) => span.attributes["gen_ai.response.id"] === "chatcmpl_1")!; + expect(first.attributes).toMatchObject({ + "gen_ai.request.model": "gpt-4.1-mini", + "gen_ai.response.model": "gpt-4.1-mini-2025-04-14", + "gen_ai.usage.input_tokens": 100, + "gen_ai.usage.output_tokens": 20, + "gen_ai.usage.cache_read.input_tokens": 40, + "gen_ai.usage.reasoning.output_tokens": 5, + }); +}); + +/** Every exported span except the root has an exported parent. */ +function expectConnected(spans: ReadableSpan[], rootName: string): void { + const ids = new Set(spans.map((span) => span.spanContext().spanId)); + const root = only(spans, named(rootName)); + const orphans = spans.filter((span) => span !== root && !ids.has(parentOf(span) ?? "")); + expect(orphans.map((span) => span.name)).toEqual([]); +} + +test("a run that throws still exports its root once flushed", async () => { + const processor = makeProcessor(); + const failing = new OpenAIResponsesModel( + scriptedClient([{ error: { message: "bad request", type: "invalid_request_error" } }], 400), + "gpt-5-mini", + ); + await expect(runCommand("true", { model: failing })).rejects.toThrow(); + + expect((await exported()).filter(named("workflow"))).toEqual([]); + await processor.forceFlush(); + const spans = await exported(); + + expectConnected(spans, "workflow"); + expect( + only(spans, (span) => span.attributes["gen_ai.operation.name"] === "chat").status.code, + ).toBe(2); +}); + +test("spans that end after their trace still reach the root", async () => { + // A processor registered first that takes time to handle each span end, such as the default + // OpenAI exporter during an export, delays this processor's span ends past the trace end. + const slow: TracingProcessor = { + async onTraceStart() {}, + async onTraceEnd() {}, + async onSpanStart() {}, + async onSpanEnd() { + await new Promise((resolve) => setTimeout(resolve, 20)); + }, + async forceFlush() {}, + async shutdown() {}, + }; + makeProcessor({}, [slow]); + await runCommand("true"); + + await vi.waitFor(async () => + expect((await exported()).filter(named("workflow"))).toHaveLength(1), + ); + const spans = await exported(); + + expectConnected(spans, "workflow"); + expect(spans.filter((span) => span.attributes["gen_ai.operation.name"] === "chat")).toHaveLength( + 2, + ); +}); + +test("an exit line in the output of a running command is not an exit code", async () => { + const processor = makeProcessor(); + const trace = new Trace({ traceId: "trace_fedcba9876543210fedcba9876543210", name: "manual" }); + await processor.onTraceStart(trace); + const span = new Span( + { + traceId: trace.traceId, + data: { + type: "function", + name: "write_stdin", + input: "{}", + output: [ + "Chunk ID: abc123", + "Wall time: 1.0000 seconds", + "Process running with session ID 3", + "Output:", + "Process exited with code 0", + ].join("\n"), + }, + }, + processor, + ); + span.start(); + span.end(); + await processor.onTraceEnd(trace); + + const tool = only(await exported(), named("write_stdin")); + expect(tool.attributes["process.exit.code"]).toBeUndefined(); +}); + +/** Starts and ends a custom span of `trace` through `processor`, the way the SDK does. */ +function customSpan( + processor: TracingProcessor, + trace: Trace, + name: string, + parent?: Span, +): Span { + const span = new Span( + { + traceId: trace.traceId, + parentId: parent?.spanId, + data: { type: "custom", name, data: {} }, + }, + processor, + ) as Span; + span.start(); + + return span; +} + +test("a flush between two runs of one trace keeps them under one root", async () => { + const processor = makeProcessor(); + await withTrace("two runs", async () => { + await runCommand("echo first"); + await processor.forceFlush(); + await runCommand("echo second"); + }); + const spans = await exported(); + + expectConnected(spans, "two runs"); + expect(new Set(spans.map((span) => span.spanContext().traceId)).size).toBe(1); +}); + +test("spans that start after shutdown still join their trace", async () => { + const processor = makeProcessor(); + const trace = new Trace({ traceId: "trace_11111111111111111111111111111111", name: "draining" }); + await processor.onTraceStart(trace); + const inFlight = customSpan(processor, trace, "in flight"); + + // A server that drains on SIGTERM keeps running after the SDK calls shutdown. + await processor.shutdown(); + const late = customSpan(processor, trace, "late", inFlight); + late.end(); + inFlight.end(); + await processor.onTraceEnd(trace); + const spans = await exported(); + + expectConnected(spans, "draining"); + const parent = only(spans, named("in flight")); + expect(parentOf(only(spans, named("late")))).toBe(parent.spanContext().spanId); +}); + +test("a span after its trace ended joins the ended root", async () => { + const processor = makeProcessor(); + const trace = new Trace({ traceId: "trace_22222222222222222222222222222222", name: "ended" }); + await processor.onTraceStart(trace); + await processor.onTraceEnd(trace); + // Such as the memory spans that a sandbox agent writes after its run. + customSpan(processor, trace, "memory flush").end(); + const spans = await exported(); + + expectConnected(spans, "ended"); + expect(new Set(spans.map((span) => span.spanContext().traceId)).size).toBe(1); +}); + +test("the root of a trace that never ends is exported after the idle wait", async () => { + vi.useFakeTimers(); + + try { + const processor = makeProcessor(); + const trace = new Trace({ traceId: "trace_33333333333333333333333333333333", name: "threw" }); + await processor.onTraceStart(trace); + customSpan(processor, trace, "step").end(); + await vi.advanceTimersByTimeAsync(59_000); + expect((await exported()).filter(named("threw"))).toEqual([]); + + await vi.advanceTimersByTimeAsync(2_000); + expectConnected(await exported(), "threw"); + } finally { + vi.useRealTimers(); + } +}); diff --git a/packages/openai-agents/tsconfig.json b/packages/openai-agents/tsconfig.json new file mode 100644 index 00000000..ff4adab5 --- /dev/null +++ b/packages/openai-agents/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["es2023"], + "moduleDetection": "force", + "module": "nodenext", + "moduleResolution": "nodenext", + "resolveJsonModule": true, + "types": ["node"], + "strict": true, + "noUnusedLocals": true, + "declaration": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + } +} diff --git a/packages/openai-agents/vite.config.ts b/packages/openai-agents/vite.config.ts new file mode 100644 index 00000000..7123f89f --- /dev/null +++ b/packages/openai-agents/vite.config.ts @@ -0,0 +1,29 @@ +import { fileURLToPath } from "node:url"; + +import { defineConfig } from "vite-plus"; +import { customExports } from "../../scripts/package-exports.mjs"; + +export default defineConfig(({ mode }) => ({ + resolve: + mode === "test" + ? { + alias: { + "@telemetry-dev/sdk": fileURLToPath(new URL("../sdk/src/index.ts", import.meta.url)), + }, + } + : undefined, + pack: { + dts: true, + exports: { + devExports: "bun", + customExports, + }, + }, + lint: { + options: { + typeAware: true, + typeCheck: true, + }, + }, + fmt: {}, +})); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f9f8d8c7..3b3ebca9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -421,6 +421,40 @@ importers: specifier: npm:@voidzero-dev/vite-plus-test@0.1.20 version: '@voidzero-dev/vite-plus-test@0.1.20(@opentelemetry/api@1.9.1)(@types/node@25.6.0)(@voidzero-dev/vite-plus-core@0.1.20(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.9.0))(esbuild@0.27.7)(jiti@2.7.0)(jsdom@28.1.0(@noble/hashes@2.2.0))(tsx@4.21.0)(typescript@5.9.3)(yaml@2.9.0)' + packages/openai-agents: + dependencies: + '@telemetry-dev/sdk': + specifier: workspace:^ + version: link:../sdk + devDependencies: + '@openai/agents': + specifier: 0.21.0 + version: 0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + '@opentelemetry/sdk-trace-base': + specifier: 'catalog:' + version: 2.12.0(@opentelemetry/api@1.9.1) + '@types/node': + specifier: ^25.5.0 + version: 25.6.0 + '@typescript/native-preview': + specifier: 7.0.0-dev.20260328.1 + version: 7.0.0-dev.20260328.1 + openai: + specifier: 'catalog:' + version: 7.18.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + typescript: + specifier: ^6.0.2 + version: 6.0.3 + vite-plus: + specifier: 0.1.20 + version: 0.1.20(@opentelemetry/api@1.9.1)(@types/node@25.6.0)(@voidzero-dev/vite-plus-core@0.1.20(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(typescript@6.0.3)(yaml@2.9.0))(esbuild@0.27.7)(jiti@2.7.0)(jsdom@28.1.0(@noble/hashes@2.2.0))(tsx@4.21.0)(typescript@6.0.3)(yaml@2.9.0) + vitest: + specifier: npm:@voidzero-dev/vite-plus-test@0.1.20 + version: '@voidzero-dev/vite-plus-test@0.1.20(@opentelemetry/api@1.9.1)(@types/node@25.6.0)(@voidzero-dev/vite-plus-core@0.1.20(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.7.0)(tsx@4.21.0)(typescript@6.0.3)(yaml@2.9.0))(esbuild@0.27.7)(jiti@2.7.0)(jsdom@28.1.0(@noble/hashes@2.2.0))(tsx@4.21.0)(typescript@6.0.3)(yaml@2.9.0)' + zod: + specifier: 'catalog:' + version: 4.4.3 + packages/opencode: dependencies: '@opentelemetry/api': @@ -543,7 +577,7 @@ importers: devDependencies: '@earendil-works/pi-coding-agent': specifier: 0.82.1 - version: 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.3)(zod@4.4.3) + version: 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.22.1)(zod@4.4.3) '@opentelemetry/sdk-logs': specifier: 'catalog:' version: 0.223.0(@opentelemetry/api@1.9.1) @@ -1562,6 +1596,29 @@ packages: resolution: {integrity: sha512-zjZfZfsVjQpQ3KML55G8MDY/HKnvZ/cPrjwcKQyyXzTk/rTGqT9KvQM3buhq8uSWIVugA/oEoC452vAa9xxQHQ==} engines: {bun: '>=1.3.14'} + '@openai/agents-core@0.21.0': + resolution: {integrity: sha512-P2jJhc85xrKpe9GUOHAVfPHqK52my49fD1CI0LnfYEvBUwZxJXQI6lgSwby9nWhvQECveM9OvBUglAXuArsHXw==} + peerDependencies: + zod: ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@openai/agents-openai@0.21.0': + resolution: {integrity: sha512-Y1u1ek6FwKx8j+T7gRYcm4kKyN0irqsyZm0XUDZ+g+8s15tu4EhNwNJ92zOHGx7Vjkr2RugWBxTeNspHjw/lQw==} + peerDependencies: + zod: ^4.0.0 + + '@openai/agents-realtime@0.21.0': + resolution: {integrity: sha512-nI0jGkSdivlXzS7HXAB1EVplaeSyoAvfM2ofHheJDJdshs3UQl3x405nmTflKWye+wQrqemK4uxL06NIhgvjHg==} + peerDependencies: + zod: ^4.0.0 + + '@openai/agents@0.21.0': + resolution: {integrity: sha512-NpJ74/dGbpYZiRYF474pCaBpxqh4A8+WHbZmaEZVQpRqieZAVH+MV4fesR9MtRGUZfpW5B91zuHzIilQp2iSxQ==} + peerDependencies: + zod: ^4.0.0 + '@opencode-ai/plugin@1.18.9': resolution: {integrity: sha512-0kFX9Usj+3N+WupIe9VnEdDNzMNbW4/C5GeIzdj02/t5kQoXsNrFpW3Br9aABebazcaYsQEWdlaLV0zQISy3OA==} peerDependencies: @@ -2184,10 +2241,12 @@ packages: '@smithy/core@3.29.1': resolution: {integrity: sha512-qoiY4nrk5OCu1+eIR1VB8l5DmON/oKiqrd5zZFAhXJXjJlLWQusKEW/SkBDAtGDcPaz86m9kfcE1lngU0GlM6A==} engines: {node: '>=18.0.0'} + deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. '@smithy/core@3.34.1': resolution: {integrity: sha512-dLcOUxz8YCv1RZUMKq6GbyUf95pLbrqh34bPvpCZ1+CByFF31BEAFewZjsGCnVsZTKdThNENfGyAgk2TJqVwSw==} engines: {node: '>=18.0.0'} + deprecated: Deprecated due to a bug in custom logger data redaction. See https://github.com/aws/aws-sdk-js-v3/issues/8393. '@smithy/credential-provider-imds@4.5.2': resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} @@ -2296,6 +2355,9 @@ packages: '@types/turndown@5.0.6': resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} + '@types/ws@8.18.2': + resolution: {integrity: sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w==} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1': resolution: {integrity: sha512-BmJGDWC0bSQ2w5O/E+Mw9eTv9RklJ3vjshu7UdD92bUMxc4V4dkBhYj5r0qxbl4f+VFNX7fXvcDDI+9o+Kb6yw==} cpu: [arm64] @@ -4488,6 +4550,18 @@ packages: utf-8-validate: optional: true + ws@8.22.1: + resolution: {integrity: sha512-oyL6OPYme/CmiFvNuGzeoxCJ4nOrZE2O1FwM9MWM1AsGri6nLXB2pg+lNA8aLeXMSav0J6eIVS1j6vUPzUtK9g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + xml-name-validator@5.0.0: resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} engines: {node: '>=18'} @@ -4946,9 +5020,9 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 - '@earendil-works/pi-agent-core@0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.3)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.22.1)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.3)(zod@4.4.3) + '@earendil-works/pi-ai': 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.22.1)(zod@4.4.3) diff: 8.0.4 ignore: 7.0.5 typebox: 1.1.38 @@ -4961,7 +5035,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.3)(zod@4.4.3)': + '@earendil-works/pi-ai@0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.22.1)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -4971,7 +5045,7 @@ snapshots: '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.21.3)(zod@4.4.3) + openai: 6.26.0(ws@8.22.1)(zod@4.4.3) partial-json: 0.1.7 typebox: 1.1.38 transitivePeerDependencies: @@ -4982,10 +5056,10 @@ snapshots: - ws - zod - '@earendil-works/pi-coding-agent@0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.3)(zod@4.4.3)': + '@earendil-works/pi-coding-agent@0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.22.1)(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.3)(zod@4.4.3) - '@earendil-works/pi-ai': 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.3)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.22.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.82.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.22.1)(zod@4.4.3) '@earendil-works/pi-tui': 0.82.1 '@silvia-odwyer/photon-node': 0.3.4 chalk: 5.6.2 @@ -5587,6 +5661,67 @@ snapshots: '@oh-my-pi/pi-utils': 17.1.8 '@oh-my-pi/pi-wire': 17.1.8 + '@openai/agents-core@0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3)': + dependencies: + '@standard-schema/spec': 1.1.0 + debug: 4.4.3 + openai: 7.18.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + optionalDependencies: + '@modelcontextprotocol/client': 2.4.0 + zod: 4.4.3 + transitivePeerDependencies: + - '@aws-sdk/credential-provider-node' + - '@smithy/hash-node' + - '@smithy/signature-v4' + - supports-color + - undici + - ws + + '@openai/agents-openai@0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3)': + dependencies: + '@openai/agents-core': 0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + openai: 7.18.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + zod: 4.4.3 + transitivePeerDependencies: + - '@aws-sdk/credential-provider-node' + - '@smithy/hash-node' + - '@smithy/signature-v4' + - supports-color + - undici + - ws + + '@openai/agents-realtime@0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(zod@4.4.3)': + dependencies: + '@openai/agents-core': 0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + '@types/ws': 8.18.2 + ws: 8.22.1 + zod: 4.4.3 + transitivePeerDependencies: + - '@aws-sdk/credential-provider-node' + - '@smithy/hash-node' + - '@smithy/signature-v4' + - bufferutil + - supports-color + - undici + - utf-8-validate + + '@openai/agents@0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3)': + dependencies: + '@openai/agents-core': 0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + '@openai/agents-openai': 0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + '@openai/agents-realtime': 0.21.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(zod@4.4.3) + openai: 7.18.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3) + zod: 4.4.3 + transitivePeerDependencies: + - '@aws-sdk/credential-provider-node' + - '@smithy/hash-node' + - '@smithy/signature-v4' + - bufferutil + - supports-color + - undici + - utf-8-validate + - ws + '@opencode-ai/plugin@1.18.9': dependencies: '@ai-sdk/provider': 3.0.8 @@ -6149,6 +6284,10 @@ snapshots: '@types/turndown@5.0.6': {} + '@types/ws@8.18.2': + dependencies: + '@types/node': 25.6.0 + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260328.1': optional: true @@ -7616,9 +7755,9 @@ snapshots: protobufjs: 7.6.5 optional: true - openai@6.26.0(ws@8.21.3)(zod@4.4.3): + openai@6.26.0(ws@8.22.1)(zod@4.4.3): optionalDependencies: - ws: 8.21.3 + ws: 8.22.1 zod: 4.4.3 openai@6.45.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(ws@8.21.3)(zod@4.4.3): @@ -7636,6 +7775,14 @@ snapshots: ws: 8.21.3 zod: 4.4.3 + openai@7.18.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.9.0)(ws@8.22.1)(zod@4.4.3): + optionalDependencies: + '@aws-sdk/credential-provider-node': 3.972.83 + '@smithy/signature-v4': 5.7.3 + undici: 8.9.0 + ws: 8.22.1 + zod: 4.4.3 + option@0.2.4: {} ow@0.28.2: @@ -7834,7 +7981,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.2 - '@types/node': 24.12.4 + '@types/node': 25.6.0 long: 5.3.2 proxy-addr@2.0.7: @@ -8526,6 +8673,8 @@ snapshots: ws@8.21.3: {} + ws@8.22.1: {} + xml-name-validator@5.0.0: optional: true diff --git a/release-please-config.json b/release-please-config.json index 79b72d61..5eaf4844 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -62,6 +62,11 @@ "component": "openai", "release-type": "node" }, + "packages/openai-agents": { + "component": "openai-agents", + "release-type": "node", + "initial-version": "0.1.0" + }, "packages/opencode": { "component": "opencode", "release-type": "node" From 2d70808bdc121d7248628b2c8a0005054af3567f Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Thu, 5 Nov 2026 15:40:51 +0000 Subject: [PATCH 2/6] docs(sdk-openai-agents): fix the sandbox example and double-count warning The sandbox example used an undefined agent, and a plain Agent ignores the sandbox client, so it now builds a SandboxAgent. The double-count warning now also covers instrumentOpenAI(), which instruments every OpenAI client in the process. --- packages/openai-agents/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/openai-agents/README.md b/packages/openai-agents/README.md index 8205196f..fd070e1d 100644 --- a/packages/openai-agents/README.md +++ b/packages/openai-agents/README.md @@ -68,12 +68,16 @@ run created its session, `sandbox.backend` (for example `e2b` or `modal`). The ` `process.exit.code` once the command has exited. A command that exits with a non-zero code is not marked as an error, because many commands, such as `grep`, exit non-zero without failing. -To also record the provider's own sandbox id, wrap the sandbox client: +To also record the provider's own sandbox id, wrap the sandbox client. The run needs a +`SandboxAgent`; a plain `Agent` ignores the sandbox client. ```ts +import { run } from "@openai/agents"; +import { SandboxAgent } from "@openai/agents/sandbox"; import { E2BSandboxClient } from "@openai/agents-extensions/sandbox/e2b"; import { instrumentSandboxClient } from "@telemetry-dev/openai-agents"; +const agent = new SandboxAgent({ name: "Coder", instructions: "Run the tests." }); const client = instrumentSandboxClient(new E2BSandboxClient()); await run(agent, "Run the tests.", { sandbox: { client } }); ``` @@ -103,6 +107,7 @@ is still live, has no such span, so the spans of that run have neither. ## Double counting -Each model call already gets a `chat` span from this processor. Do not also pass a client wrapped -with `wrapOpenAI` from `@telemetry-dev/openai` to `setDefaultOpenAIClient`: each model call would -then be counted twice. +Each model call already gets a `chat` span from this processor. Do not also instrument the OpenAI +client that the Agents SDK uses with `@telemetry-dev/openai`: neither pass a client wrapped with +`wrapOpenAI` to `setDefaultOpenAIClient`, nor call `instrumentOpenAI()`, which instruments every +client in the process. Each model call would then be counted twice. From 39dd66e61951d7bf910ada0d2206c166aa6ae11e Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Thu, 5 Nov 2026 16:21:23 +0000 Subject: [PATCH 3/6] docs(sdk-openai-agents): say when runs share a trace The intro said each run is one trace. Runs that share a groupId land in one session trace, a run started inside an active span joins that span's trace, and sessionMode process groups runs without a groupId. --- packages/openai-agents/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/openai-agents/README.md b/packages/openai-agents/README.md index fd070e1d..5031b5a1 100644 --- a/packages/openai-agents/README.md +++ b/packages/openai-agents/README.md @@ -1,8 +1,15 @@ # @telemetry-dev/openai-agents OpenAI Agents SDK integration for [telemetry.dev](https://telemetry.dev). It sends each agent run -to telemetry.dev as one trace: the agent and its turns, each model call with its token usage, each -tool call, and each command that a sandbox agent ran, with its exit code and the sandbox it ran in. +to telemetry.dev: the agent and its turns, each model call with its token usage, each tool call, and +each command that a sandbox agent ran, with its exit code and the sandbox it ran in. + +Each run is one trace, unless it starts inside an active OpenTelemetry span, such as an HTTP request +span, and joins that span's trace. Runs that share a `groupId` share a session. Outside an active +span, they land in +[one trace per session](https://telemetry.dev/docs/sdk/typescript#one-trace-per-session), each run +as a root span. With `sessionMode: "process"`, runs without a `groupId` share the process's session +in the same way. ## Install From 33c6df0c947ff58f8737802c0c44a2b6bd8db6a6 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Thu, 5 Nov 2026 16:42:42 +0000 Subject: [PATCH 4/6] docs(sdk-openai-agents): list how runs group into traces Match the docs guide: cover propagateAttributes() sessions and set sessionMode on the processor. --- packages/openai-agents/README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/openai-agents/README.md b/packages/openai-agents/README.md index 5031b5a1..b631a10b 100644 --- a/packages/openai-agents/README.md +++ b/packages/openai-agents/README.md @@ -4,12 +4,22 @@ OpenAI Agents SDK integration for [telemetry.dev](https://telemetry.dev). It sen to telemetry.dev: the agent and its turns, each model call with its token usage, each tool call, and each command that a sandbox agent ran, with its exit code and the sandbox it ran in. -Each run is one trace, unless it starts inside an active OpenTelemetry span, such as an HTTP request -span, and joins that span's trace. Runs that share a `groupId` share a session. Outside an active -span, they land in -[one trace per session](https://telemetry.dev/docs/sdk/typescript#one-trace-per-session), each run -as a root span. With `sessionMode: "process"`, runs without a `groupId` share the process's session -in the same way. +How runs group into traces: + +- A run is one trace by default. +- A run that starts inside an active OpenTelemetry span, such as an HTTP request span, joins that + span's trace. +- Runs that share a `groupId` share a session: the processor sends the `groupId` as the session ID + (`gen_ai.conversation.id`). Runs inside the same + [`propagateAttributes({ sessionId })`](https://telemetry.dev/docs/sdk/typescript#add-user-session-and-metadata) + session (from `@telemetry-dev/sdk`) do too; a `groupId` takes precedence. Outside an active span, + the runs of a session land in + [one trace per session](https://telemetry.dev/docs/sdk/typescript#one-trace-per-session), each run + as a root span. +- With `telemetryDevProcessor({ sessionMode: "process" })`, runs without a session of their own + share the process's + [generated session](https://telemetry.dev/docs/sdk/typescript#process-sessions) in the same way. + Use this in a CLI or script that holds one conversation per process, not in a long-running server. ## Install From fbc63ed35dd478fdbe21d8c4fbca6a2d0c1b7e64 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Thu, 5 Nov 2026 16:54:38 +0000 Subject: [PATCH 5/6] docs(sdk-openai-agents): list every session source sent with capture off Name propagateAttributes() and process sessions, and the user ID and metadata that propagateAttributes() sends. --- packages/openai-agents/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/openai-agents/README.md b/packages/openai-agents/README.md index b631a10b..5e168dd1 100644 --- a/packages/openai-agents/README.md +++ b/packages/openai-agents/README.md @@ -118,9 +118,10 @@ is still live, has no such span, so the spans of that run have neither. `setTraceProcessors` to stop that. - With capture off, telemetry.dev still receives span names and timing, model names, token usage, agent, tool, handoff, and guardrail names, the MCP server of a tool call, exit codes, sandbox - backends and ids, error types, the trace's `groupId` as the session id, and the string, number, - and boolean values of trace metadata and of the data of custom spans. Metadata does not go - through `mask`. + backends and ids, error types, the session id (such as the trace's `groupId`, a + `propagateAttributes()` session, or the process session), the user id and metadata of + `propagateAttributes()`, and the string, number, and boolean values of trace metadata and of the + data of custom spans. Metadata does not go through `mask`. ## Double counting From 2adf8a01a462fce4723bc34de7e6d724039da93c Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Thu, 5 Nov 2026 17:19:34 +0000 Subject: [PATCH 6/6] test(sdk-openai-agents): cover how runs group into traces Test the rules the README states through a real Runner: groupId and propagateAttributes() sessions share one session trace with groupId winning, a run inside an active span joins that span, and process sessions group runs without a session. In the SDK, test that a root inside a span of the host's own context manager joins that span. --- .../openai-agents/tests/openai-agents.test.ts | 89 ++++++++++++++++++- packages/sdk/tests/propagate.test.ts | 37 +++++++- 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/packages/openai-agents/tests/openai-agents.test.ts b/packages/openai-agents/tests/openai-agents.test.ts index 811bf6d6..e4098b79 100644 --- a/packages/openai-agents/tests/openai-agents.test.ts +++ b/packages/openai-agents/tests/openai-agents.test.ts @@ -1,4 +1,5 @@ import { + Agent, type Model, OpenAIChatCompletionsModel, OpenAIResponsesModel, @@ -14,7 +15,7 @@ import { import { SandboxAgent, shell } from "@openai/agents/sandbox"; import { UnixLocalSandboxClient } from "@openai/agents/sandbox/local"; import { InMemorySpanExporter, type ReadableSpan } from "@opentelemetry/sdk-trace-base"; -import { flush } from "@telemetry-dev/sdk"; +import { flush, propagateAttributes, startActiveSpan } from "@telemetry-dev/sdk"; import OpenAI from "openai"; import { afterEach, beforeEach, expect, test, vi } from "vitest"; @@ -340,6 +341,92 @@ test("two concurrent runs produce two traces whose spans do not cross", async () } }); +/** One run of an agent that answers "done", with its root span named `workflowName`. */ +async function runAgent(workflowName: string, groupId?: string): Promise { + const model = new OpenAIResponsesModel( + scriptedClient([ + response("resp_1", [message("done")], { input_tokens: 1, output_tokens: 1, total_tokens: 2 }), + ]), + "gpt-5-mini", + ); + + await new Runner({ workflowName, groupId }).run(new Agent({ name: "assistant", model }), "go"); +} + +const traceOf = (span: ReadableSpan): string => span.spanContext().traceId; +const sessionOf = (span: ReadableSpan): unknown => span.attributes["gen_ai.conversation.id"]; + +test("runs that share a groupId land in one session trace", async () => { + makeProcessor(); + await runAgent("a1", "g1"); + await runAgent("a2", "g1"); + await runAgent("b", "g2"); + await runAgent("c"); + const spans = await exported(); + const [a1, a2, b, c] = ["a1", "a2", "b", "c"].map((name) => only(spans, named(name))); + + expect(traceOf(a2)).toBe(traceOf(a1)); + expect(new Set([a1, b, c].map(traceOf)).size).toBe(3); + expect([a1, a2, b, c].map(sessionOf)).toEqual(["g1", "g1", "g2", undefined]); + // Each run of a session is a root under one parent that is never exported. + expect(parentOf(a1)).toEqual(expect.any(String)); + expect(parentOf(a2)).toBe(parentOf(a1)); + expect(spans.map((span) => span.spanContext().spanId)).not.toContain(parentOf(a1)); + expect(parentOf(c)).toBeUndefined(); +}); + +test("a run inside an active span joins that span's trace, even with a groupId", async () => { + makeProcessor(); + await startActiveSpan("request", async () => { + await runAgent("a", "g1"); + await runAgent("b", "g1"); + }); + const spans = await exported(); + const request = only(spans, named("request")); + + for (const root of [only(spans, named("a")), only(spans, named("b"))]) { + expect(traceOf(root)).toBe(traceOf(request)); + expect(parentOf(root)).toBe(request.spanContext().spanId); + expect(sessionOf(root)).toBe("g1"); + } +}); + +test("runs share a propagateAttributes session, and a groupId takes precedence", async () => { + makeProcessor(); + await propagateAttributes({ sessionId: "s1" }, async () => { + await runAgent("a"); + await runAgent("b"); + await runAgent("c", "g1"); + await runAgent("d", "g1"); + }); + const spans = await exported(); + const [a, b, c, d] = ["a", "b", "c", "d"].map((name) => only(spans, named(name))); + + expect(traceOf(b)).toBe(traceOf(a)); + expect(traceOf(d)).toBe(traceOf(c)); + expect(traceOf(c)).not.toBe(traceOf(a)); + expect([a, b, c, d].map(sessionOf)).toEqual(["s1", "s1", "g1", "g1"]); + const groupTraceSpans = spans.filter((span) => traceOf(span) === traceOf(c)); + expect(new Set(groupTraceSpans.map(sessionOf))).toEqual(new Set(["g1"])); +}); + +test("process sessions group the runs that have no session of their own", async () => { + makeProcessor({ sessionMode: "process" }); + await runAgent("a"); + await runAgent("b"); + await runAgent("c", "g1"); + await runAgent("d", "g1"); + const spans = await exported(); + const [a, b, c, d] = ["a", "b", "c", "d"].map((name) => only(spans, named(name))); + + expect(traceOf(b)).toBe(traceOf(a)); + expect(traceOf(d)).toBe(traceOf(c)); + expect(traceOf(c)).not.toBe(traceOf(a)); + expect(sessionOf(a)).toEqual(expect.any(String)); + expect(sessionOf(b)).toBe(sessionOf(a)); + expect(sessionOf(c)).toBe("g1"); +}); + test("a span whose parent is unknown nests under its trace root", async () => { const processor = makeProcessor(); const trace = new Trace({ traceId: "trace_0123456789abcdef0123456789abcdef", name: "manual" }); diff --git a/packages/sdk/tests/propagate.test.ts b/packages/sdk/tests/propagate.test.ts index bb1a9f2c..0318d402 100644 --- a/packages/sdk/tests/propagate.test.ts +++ b/packages/sdk/tests/propagate.test.ts @@ -1,4 +1,13 @@ -import { ROOT_CONTEXT, SpanKind, TraceFlags } from "@opentelemetry/api"; +import { AsyncLocalStorage } from "node:async_hooks"; + +import { + type Context, + context, + ROOT_CONTEXT, + SpanKind, + trace, + TraceFlags, +} from "@opentelemetry/api"; import { AlwaysOffSampler, AlwaysOnSampler, @@ -6,7 +15,7 @@ import { SamplingDecision, TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; -import { sessionSpanContext } from "@telemetry-dev/otel"; +import { AlsContextManager, sessionSpanContext } from "@telemetry-dev/otel"; import { afterEach, expect, test } from "vitest"; import { TRUNCATION_MARKER } from "../src/capture.ts"; @@ -180,6 +189,30 @@ test("nested spans keep their real parent inside a session", async () => { expect(root.parentSpanContext?.spanId).toBe(sessionSpanContext("td_live_test", "s1").spanId); }); +test("a session root inside a span of the host's own context manager joins that span", async () => { + const { spans } = setup(); + // A host OTel setup (NodeSDK, HTTP instrumentation) keeps its spans in its own store, not ours. + const host = { + traceId: "0af7651916cd43dd8448eb211c80319c", + spanId: "b7ad6b7169203331", + traceFlags: TraceFlags.SAMPLED, + }; + const manager = new AlsContextManager(new AsyncLocalStorage()); + expect(context.setGlobalContextManager(manager)).toBe(true); + + try { + context.with(trace.setSpanContext(ROOT_CONTEXT, host), () => { + startSpan("run", { attributes: { "gen_ai.conversation.id": "s1" } }).end(); + }); + } finally { + context.disable(); + } + await flush(); + const run = spans.getFinishedSpans().find((s) => s.name === "run")!; + expect(run.spanContext().traceId).toBe(host.traceId); + expect(run.parentSpanContext?.spanId).toBe(host.spanId); +}); + test("spans without a session id keep separate random traces", async () => { const { spans } = setup(); startSpan("a").end();