diff --git a/A365_DOCUMENTATION.md b/A365_DOCUMENTATION.md index 7b041e07..ee2e6752 100644 --- a/A365_DOCUMENTATION.md +++ b/A365_DOCUMENTATION.md @@ -25,8 +25,16 @@ import { const invokeScope = InvokeAgentScope.start( { conversationId: "conv-123", sessionId: "session-456" }, - {}, - { agentId: "agent-1", tenantId: "tenant-1" }, + { + requestParameters: { + model: "gpt-4o", + outputType: "json", + systemInstructions: [ + { type: "text", content: "You are a helpful assistant." }, + ], + }, + }, + { agentId: "agent-1", tenantId: "tenant-1", providerName: "openai" }, ); invokeScope.run(async () => { @@ -46,9 +54,43 @@ invokeScope.run(async () => { inferenceScope.dispose(); }); +invokeScope.recordResponseParameters({ + finishReasons: ["stop"], + inputTokens: 120, + outputTokens: 42, + cacheWriteInputTokens: 10, + cacheReadInputTokens: 8, +}); invokeScope.dispose(); ``` +`InvokeAgentScope.start()` captures request parameters immediately, while `recordResponseParameters()` +captures response and usage values after the agent completes. + +| Input field | Emitted attribute key | +| --- | --- | +| `requestParameters.model` | `gen_ai.request.model` | +| `requestParameters.seed` | `gen_ai.request.seed` | +| `requestParameters.choiceCount` | `gen_ai.request.choice.count` | +| `requestParameters.frequencyPenalty` | `gen_ai.request.frequency_penalty` | +| `requestParameters.maxTokens` | `gen_ai.request.max_tokens` | +| `requestParameters.presencePenalty` | `gen_ai.request.presence_penalty` | +| `requestParameters.stopSequences` | `gen_ai.request.stop_sequences` | +| `requestParameters.temperature` | `gen_ai.request.temperature` | +| `requestParameters.topP` | `gen_ai.request.top_p` | +| `requestParameters.dataSourceId` | `gen_ai.data_source.id` | +| `requestParameters.outputType` | `gen_ai.output.type` | +| `requestParameters.systemInstructions` | `gen_ai.system_instructions` (JSON-serialized parts array) | +| `responseParameters.finishReasons` | `gen_ai.response.finish_reasons` | +| `responseParameters.inputTokens` | `gen_ai.usage.input_tokens` | +| `responseParameters.outputTokens` | `gen_ai.usage.output_tokens` | +| `responseParameters.cacheWriteInputTokens` | `gen_ai.usage.cache_write.input_tokens` | +| `responseParameters.cacheReadInputTokens` | `gen_ai.usage.cache_read.input_tokens` | +| `agentDetails.providerName` | `gen_ai.provider.name` | + +System instructions may contain sensitive content. Only capture them when you +intend to store prompt text and have reviewed downstream access controls. + ## Baggage And Context Use `BaggageBuilder` when you want tenant, agent, user, conversation, or session data to flow with the active context. diff --git a/CHANGELOG.md b/CHANGELOG.md index d61c5f77..07930907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Features Added +- Add GenAI v1.42 InvokeAgent request, response, cache-token, and provider attribute capture for manual A365 scopes. [#239](https://github.com/microsoft/opentelemetry-distro-javascript/pull/239) + ### Other Changes - Consolidate Dependabot updates for Vitest 4.1.11, Hono 4.13.7, qs 6.16.0, fast-uri 3.1.7, actions/deploy-pages 5.0.1, and actions/checkout 7.0.1. diff --git a/src/a365/constants.ts b/src/a365/constants.ts index 94f28a98..9255703a 100644 --- a/src/a365/constants.ts +++ b/src/a365/constants.ts @@ -67,8 +67,16 @@ export class OpenTelemetryConstants { /** Attribute key for the gen-ai operation name (`gen_ai.operation.name`). */ public static readonly GEN_AI_OPERATION_NAME_KEY = "gen_ai.operation.name"; + /** Attribute key for the external data source identifier (`gen_ai.data_source.id`). */ + public static readonly GEN_AI_DATA_SOURCE_ID_KEY = "gen_ai.data_source.id"; + /** Attribute key for the requested output type (`gen_ai.output.type`). */ + public static readonly GEN_AI_OUTPUT_TYPE_KEY = "gen_ai.output.type"; /** Attribute key for the requested model name (`gen_ai.request.model`). */ public static readonly GEN_AI_REQUEST_MODEL_KEY = "gen_ai.request.model"; + /** Attribute key for the requested number of choices (`gen_ai.request.choice.count`). */ + public static readonly GEN_AI_REQUEST_CHOICE_COUNT_KEY = "gen_ai.request.choice.count"; + /** Attribute key for the frequency penalty (`gen_ai.request.frequency_penalty`). */ + public static readonly GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY = "gen_ai.request.frequency_penalty"; /** Attribute key for the model that produced the response (`gen_ai.response.model`). */ public static readonly GEN_AI_RESPONSE_MODEL_KEY = "gen_ai.response.model"; /** Attribute key for the finish reasons returned by the model (`gen_ai.response.finish_reasons`). */ @@ -77,6 +85,12 @@ export class OpenTelemetryConstants { public static readonly GEN_AI_PROVIDER_NAME_KEY = "gen_ai.provider.name"; /** Attribute key for the requested maximum number of tokens (`gen_ai.request.max_tokens`). */ public static readonly GEN_AI_REQUEST_MAX_TOKENS_KEY = "gen_ai.request.max_tokens"; + /** Attribute key for the presence penalty (`gen_ai.request.presence_penalty`). */ + public static readonly GEN_AI_REQUEST_PRESENCE_PENALTY_KEY = "gen_ai.request.presence_penalty"; + /** Attribute key for the request seed (`gen_ai.request.seed`). */ + public static readonly GEN_AI_REQUEST_SEED_KEY = "gen_ai.request.seed"; + /** Attribute key for the stop sequences (`gen_ai.request.stop_sequences`). */ + public static readonly GEN_AI_REQUEST_STOP_SEQUENCES_KEY = "gen_ai.request.stop_sequences"; /** Attribute key for the sampling temperature (`gen_ai.request.temperature`). */ public static readonly GEN_AI_REQUEST_TEMPERATURE_KEY = "gen_ai.request.temperature"; /** Attribute key for the nucleus-sampling top-p value (`gen_ai.request.top_p`). */ @@ -94,6 +108,12 @@ export class OpenTelemetryConstants { // ── GenAI usage ────────────────────────────────────────────────── + /** Attribute key for the number of input tokens written into the cache (`gen_ai.usage.cache_write.input_tokens`). */ + public static readonly GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY = + "gen_ai.usage.cache_write.input_tokens"; + /** Attribute key for the number of input tokens read from the cache (`gen_ai.usage.cache_read.input_tokens`). */ + public static readonly GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY = + "gen_ai.usage.cache_read.input_tokens"; /** Attribute key for the number of input (prompt) tokens (`gen_ai.usage.input_tokens`). */ public static readonly GEN_AI_USAGE_INPUT_TOKENS_KEY = "gen_ai.usage.input_tokens"; /** Attribute key for the number of output (completion) tokens (`gen_ai.usage.output_tokens`). */ diff --git a/src/a365/contracts.ts b/src/a365/contracts.ts index 39bcba91..25166c6c 100644 --- a/src/a365/contracts.ts +++ b/src/a365/contracts.ts @@ -203,6 +203,9 @@ export interface GenericPart { [key: string]: unknown; } +/** Content part accepted for system instructions. */ +export type SystemInstructionPart = TextPart | GenericPart; + /** Union of all message part types per OTEL gen-ai semantic conventions. */ export type MessagePart = | TextPart @@ -363,10 +366,56 @@ export interface ServiceEndpoint { // Scope detail types // --------------------------------------------------------------------------- +/** Request-side GenAI parameters captured for agent invocation telemetry. */ +export interface GenAiRequestParameters { + /** Name of the requested model. */ + model?: string; + /** Seed used to make sampling reproducible. */ + seed?: number; + /** Number of response choices requested from the model. */ + choiceCount?: number; + /** Frequency penalty applied during token sampling. */ + frequencyPenalty?: number; + /** Maximum number of tokens requested for generation. */ + maxTokens?: number; + /** Presence penalty applied during token sampling. */ + presencePenalty?: number; + /** Stop sequences supplied with the request. */ + stopSequences?: string[]; + /** Sampling temperature for the request. */ + temperature?: number; + /** Nucleus-sampling top-p value for the request. */ + topP?: number; + /** Identifier of the external data source used to ground the request. */ + dataSourceId?: string; + /** Requested output type (for example, `json`). */ + outputType?: string; + /** Structured system instructions provided to the model. */ + systemInstructions?: SystemInstructionPart[]; +} + +/** Response-side GenAI parameters captured for agent invocation telemetry. */ +export interface GenAiResponseParameters { + /** Finish reasons returned by the model. */ + finishReasons?: string[]; + /** Number of input (prompt) tokens consumed by the response. */ + inputTokens?: number; + /** Number of output (completion) tokens produced by the response. */ + outputTokens?: number; + /** Number of input tokens written into the cache. */ + cacheWriteInputTokens?: number; + /** Number of input tokens read from the cache. */ + cacheReadInputTokens?: number; +} + /** Details for invoking agent scope. */ export interface InvokeAgentScopeDetails { /** Endpoint the agent is being invoked on. */ endpoint?: ServiceEndpoint; + /** Request-side GenAI parameters associated with the invoke-agent span. */ + requestParameters?: GenAiRequestParameters; + /** Response-side GenAI parameters associated with the invoke-agent span. */ + responseParameters?: GenAiResponseParameters; } /** Details of a tool call made by an agent. */ diff --git a/src/a365/index.ts b/src/a365/index.ts index 9ae02297..a34b255c 100644 --- a/src/a365/index.ts +++ b/src/a365/index.ts @@ -51,6 +51,7 @@ export type { OutputMessagesParam, ResponseMessagesParam, MessagePart, + SystemInstructionPart, TextPart, ToolCallRequestPart, ToolCallResponsePart, @@ -61,6 +62,8 @@ export type { Request, Channel, ServiceEndpoint, + GenAiRequestParameters, + GenAiResponseParameters, InvokeAgentScopeDetails, ToolCallDetails, InferenceDetails, diff --git a/src/a365/message-utils.ts b/src/a365/message-utils.ts index 0b76af69..b3fd6cf0 100644 --- a/src/a365/message-utils.ts +++ b/src/a365/message-utils.ts @@ -14,6 +14,7 @@ import type { OutputMessages, InputMessagesParam, OutputMessagesParam, + SystemInstructionPart, } from "./contracts.js"; import { MessageRole, DEFAULT_FINISH_REASON } from "./contracts.js"; @@ -105,6 +106,25 @@ export function serializeMessages(wrapper: InputMessages | OutputMessages): stri } } +/** + * Serializes system instruction parts to a JSON array. + * + * The fallback keeps telemetry recording non-throwing when a part contains + * non-JSON-serializable values. + */ +export function serializeSystemInstructions(parts: SystemInstructionPart[]): string { + try { + return JSON.stringify(parts); + } catch { + return JSON.stringify([ + { + type: "text", + content: `[serialization failed: ${parts.length} ${parts.length === 1 ? "instruction" : "instructions"}]`, + }, + ]); + } +} + /** * Ensures the value is always a JSON-parseable string. * - Objects are serialized via JSON.stringify. diff --git a/src/a365/scopes/InvokeAgentScope.ts b/src/a365/scopes/InvokeAgentScope.ts index bcfda81b..b64c3e74 100644 --- a/src/a365/scopes/InvokeAgentScope.ts +++ b/src/a365/scopes/InvokeAgentScope.ts @@ -4,12 +4,15 @@ import { SpanKind } from "@opentelemetry/api"; import { OpenTelemetryScope } from "./OpenTelemetryScope.js"; import { OpenTelemetryConstants } from "../constants.js"; +import { serializeSystemInstructions } from "../message-utils.js"; import type { InvokeAgentScopeDetails, CallerDetails, Request, SpanDetails, AgentDetails, + GenAiRequestParameters, + GenAiResponseParameters, InputMessagesParam, OutputMessagesParam, } from "../contracts.js"; @@ -69,9 +72,6 @@ export class InvokeAgentScope extends OpenTelemetryScope { callerDetails?.userDetails, ); - // Provider name - this.setTagMaybe(OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY, agentDetails.providerName); - // Session ID this.setTagMaybe(OpenTelemetryConstants.SESSION_ID_KEY, request.sessionId); @@ -97,6 +97,14 @@ export class InvokeAgentScope extends OpenTelemetryScope { this.recordInputMessages(request.content); } + if (invokeScopeDetails.requestParameters) { + this.mapRequestParameters(invokeScopeDetails.requestParameters); + } + + if (invokeScopeDetails.responseParameters) { + this.mapResponseParameters(invokeScopeDetails.responseParameters); + } + // Caller agent details for A2A scenarios const callerAgent = callerDetails?.callerAgentDetails; if (callerAgent) { @@ -130,6 +138,10 @@ export class InvokeAgentScope extends OpenTelemetryScope { this.recordOutputMessages(response); } + public recordResponseParameters(responseParameters: GenAiResponseParameters): void { + this.mapResponseParameters(responseParameters); + } + /** Records the input messages for telemetry tracking. */ public override recordInputMessages(messages: InputMessagesParam): void { super.recordInputMessages(messages); @@ -139,4 +151,68 @@ export class InvokeAgentScope extends OpenTelemetryScope { public override recordOutputMessages(messages: OutputMessagesParam): void { super.recordOutputMessages(messages); } + + private mapRequestParameters(requestParameters: GenAiRequestParameters): void { + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY, requestParameters.model); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_SEED_KEY, requestParameters.seed); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_REQUEST_CHOICE_COUNT_KEY, + requestParameters.choiceCount, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY, + requestParameters.frequencyPenalty, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_REQUEST_MAX_TOKENS_KEY, + requestParameters.maxTokens, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_REQUEST_PRESENCE_PENALTY_KEY, + requestParameters.presencePenalty, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_REQUEST_STOP_SEQUENCES_KEY, + requestParameters.stopSequences, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_REQUEST_TEMPERATURE_KEY, + requestParameters.temperature, + ); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_REQUEST_TOP_P_KEY, requestParameters.topP); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_DATA_SOURCE_ID_KEY, + requestParameters.dataSourceId, + ); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_OUTPUT_TYPE_KEY, requestParameters.outputType); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_SYSTEM_INSTRUCTIONS_KEY, + requestParameters.systemInstructions === undefined + ? undefined + : serializeSystemInstructions(requestParameters.systemInstructions), + ); + } + + private mapResponseParameters(responseParameters: GenAiResponseParameters): void { + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY, + responseParameters.finishReasons, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY, + responseParameters.inputTokens, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY, + responseParameters.outputTokens, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + responseParameters.cacheWriteInputTokens, + ); + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + responseParameters.cacheReadInputTokens, + ); + } } diff --git a/src/a365/scopes/OpenTelemetryScope.ts b/src/a365/scopes/OpenTelemetryScope.ts index 0856604d..80cf2ce9 100644 --- a/src/a365/scopes/OpenTelemetryScope.ts +++ b/src/a365/scopes/OpenTelemetryScope.ts @@ -121,6 +121,7 @@ export abstract class OpenTelemetryScope { agentDetails.agentBlueprintId, ); this.setTagMaybe(OpenTelemetryConstants.GEN_AI_AGENT_VERSION_KEY, agentDetails.agentVersion); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY, agentDetails.providerName); } // Set tenant ID diff --git a/src/index.ts b/src/index.ts index a2869127..148f09a1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -70,6 +70,8 @@ export type { Request as A365Request, Channel, ServiceEndpoint, + GenAiRequestParameters, + GenAiResponseParameters, InvokeAgentScopeDetails, ToolCallDetails, InferenceDetails, @@ -86,6 +88,7 @@ export type { OutputMessagesParam, ResponseMessagesParam, MessagePart, + SystemInstructionPart, TextPart, ToolCallRequestPart, ToolCallResponsePart, diff --git a/test/internal/unit/a365/invokeAgentGenAiParameters.test.ts b/test/internal/unit/a365/invokeAgentGenAiParameters.test.ts new file mode 100644 index 00000000..7fce4b17 --- /dev/null +++ b/test/internal/unit/a365/invokeAgentGenAiParameters.test.ts @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { assert, describe, it } from "vitest"; +import { OpenTelemetryConstants } from "../../../../src/index.js"; +import type { + GenAiRequestParameters as RootGenAiRequestParameters, + GenAiResponseParameters as RootGenAiResponseParameters, + InvokeAgentScopeDetails as RootInvokeAgentScopeDetails, + SystemInstructionPart as RootSystemInstructionPart, +} from "../../../../src/index.js"; +import type { + GenAiRequestParameters, + GenAiResponseParameters, + InvokeAgentScopeDetails, + ServiceEndpoint, + SystemInstructionPart, +} from "../../../../src/a365/index.js"; + +type Expect = T; +type Equal = + (() => Candidate extends Left ? 1 : 2) extends () => Candidate extends Right + ? 1 + : 2 + ? true + : false; +type IsOptional = Omit extends T ? true : false; + +type _RootRequestParametersExportMatchesA365 = Expect< + Equal +>; +type _RootResponseParametersExportMatchesA365 = Expect< + Equal +>; +type _RootSystemInstructionPartExportMatchesA365 = Expect< + Equal +>; +type _RequestModelIsOptionalString = Expect< + Equal +>; +type _RequestSeedIsOptionalNumber = Expect< + Equal +>; +type _RequestChoiceCountIsOptionalNumber = Expect< + Equal +>; +type _RequestFrequencyPenaltyIsOptionalNumber = Expect< + Equal +>; +type _RequestMaxTokensIsOptionalNumber = Expect< + Equal +>; +type _RequestPresencePenaltyIsOptionalNumber = Expect< + Equal +>; +type _RequestStopSequencesIsOptionalStringArray = Expect< + Equal +>; +type _RequestTemperatureIsOptionalNumber = Expect< + Equal +>; +type _RequestTopPIsOptionalNumber = Expect< + Equal +>; +type _RequestDataSourceIdIsOptionalString = Expect< + Equal +>; +type _RequestOutputTypeIsOptionalString = Expect< + Equal +>; +type _RequestSystemInstructionsIsOptionalInstructionParts = Expect< + Equal +>; +type _ResponseFinishReasonsIsOptionalStringArray = Expect< + Equal +>; +type _ResponseInputTokensIsOptionalNumber = Expect< + Equal +>; +type _ResponseOutputTokensIsOptionalNumber = Expect< + Equal +>; +type _ResponseCacheWriteInputTokensIsOptionalNumber = Expect< + Equal +>; +type _ResponseCacheReadInputTokensIsOptionalNumber = Expect< + Equal +>; +type _InvokeAgentScopeDetailsEndpointIsPreserved = Expect< + Equal +>; +type _InvokeAgentScopeDetailsHasOptionalRequestParameters = Expect< + IsOptional +>; +type _InvokeAgentScopeDetailsHasOptionalResponseParameters = Expect< + IsOptional +>; + +describe("InvokeAgent GenAI parameter contracts", () => { + it("accepts request and response parameters on invoke agent scope details", () => { + const requestParameters: RootGenAiRequestParameters = { + model: "gpt-4.1", + seed: 42, + choiceCount: 2, + frequencyPenalty: 0.25, + maxTokens: 512, + presencePenalty: -0.5, + stopSequences: ["DONE", "STOP"], + temperature: 0.2, + topP: 0.8, + dataSourceId: "sharepoint", + outputType: "json", + systemInstructions: [{ type: "text", content: "Answer with JSON only." }], + }; + const responseParameters: RootGenAiResponseParameters = { + finishReasons: ["stop"], + inputTokens: 120, + outputTokens: 48, + cacheWriteInputTokens: 12, + cacheReadInputTokens: 3, + }; + const scopeDetails: RootInvokeAgentScopeDetails = { + endpoint: { host: "agents.contoso.com", port: 443, protocol: "https" }, + requestParameters, + responseParameters, + }; + + assert.deepStrictEqual(scopeDetails.requestParameters, requestParameters); + assert.deepStrictEqual(scopeDetails.responseParameters, responseParameters); + assert.strictEqual(scopeDetails.endpoint?.host, "agents.contoso.com"); + }); + + it("defines invoke-agent GenAI semantic-convention constants", () => { + assert.strictEqual(OpenTelemetryConstants.GEN_AI_DATA_SOURCE_ID_KEY, "gen_ai.data_source.id"); + assert.strictEqual(OpenTelemetryConstants.GEN_AI_OUTPUT_TYPE_KEY, "gen_ai.output.type"); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_REQUEST_CHOICE_COUNT_KEY, + "gen_ai.request.choice.count", + ); + assert.strictEqual(OpenTelemetryConstants.GEN_AI_REQUEST_SEED_KEY, "gen_ai.request.seed"); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY, + "gen_ai.request.frequency_penalty", + ); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_REQUEST_PRESENCE_PENALTY_KEY, + "gen_ai.request.presence_penalty", + ); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_REQUEST_STOP_SEQUENCES_KEY, + "gen_ai.request.stop_sequences", + ); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY, + "gen_ai.usage.cache_write.input_tokens", + ); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY, + "gen_ai.usage.cache_read.input_tokens", + ); + assert.strictEqual(OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY, "gen_ai.request.model"); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_REQUEST_MAX_TOKENS_KEY, + "gen_ai.request.max_tokens", + ); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_REQUEST_TEMPERATURE_KEY, + "gen_ai.request.temperature", + ); + assert.strictEqual(OpenTelemetryConstants.GEN_AI_REQUEST_TOP_P_KEY, "gen_ai.request.top_p"); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_SYSTEM_INSTRUCTIONS_KEY, + "gen_ai.system_instructions", + ); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY, + "gen_ai.response.finish_reasons", + ); + assert.strictEqual(OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY, "gen_ai.provider.name"); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY, + "gen_ai.usage.input_tokens", + ); + assert.strictEqual( + OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY, + "gen_ai.usage.output_tokens", + ); + }); +}); diff --git a/test/internal/unit/a365/scopes.test.ts b/test/internal/unit/a365/scopes.test.ts index 4c51f71e..b8b1099a 100644 --- a/test/internal/unit/a365/scopes.test.ts +++ b/test/internal/unit/a365/scopes.test.ts @@ -21,11 +21,14 @@ import { } from "../../../../src/a365/index.js"; import type { AgentDetails, + GenAiRequestParameters, + GenAiResponseParameters, InvokeAgentScopeDetails, ToolCallDetails, InferenceDetails, UserDetails, OutputResponse, + SystemInstructionPart, } from "../../../../src/a365/index.js"; import { InferenceOperationType, MessageRole } from "../../../../src/a365/index.js"; import { safeSerializeToJson } from "../../../../src/a365/message-utils.js"; @@ -1147,6 +1150,227 @@ describe("Request content and message serialization (span attributes)", () => { }); }); + describe("InvokeAgentScope – GenAI request and response parameters", () => { + it("should record all request attributes and response-at-start attributes", () => { + const requestParameters: GenAiRequestParameters = { + model: "gpt-4.1", + seed: 42, + choiceCount: 2, + frequencyPenalty: 0.25, + maxTokens: 512, + presencePenalty: -0.5, + stopSequences: ["DONE", "STOP"], + temperature: 0.2, + topP: 0.8, + dataSourceId: "sharepoint", + outputType: "json", + systemInstructions: [{ type: "text", content: "Answer with JSON only." }], + }; + const responseParameters: GenAiResponseParameters = { + finishReasons: ["stop"], + inputTokens: 120, + outputTokens: 48, + cacheWriteInputTokens: 12, + cacheReadInputTokens: 3, + }; + const scope = InvokeAgentScope.start( + testRequest, + { requestParameters, responseParameters }, + { ...testAgentDetails, providerName: "azure-openai" }, + ); + scope.dispose(); + + const attributes = getLastSpan().attributes; + expect(attributes[OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY]).toBe("azure-openai"); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY]).toBe("gpt-4.1"); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_SEED_KEY]).toBe(42); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_CHOICE_COUNT_KEY]).toBe(2); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY]).toBe(0.25); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_MAX_TOKENS_KEY]).toBe(512); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_PRESENCE_PENALTY_KEY]).toBe(-0.5); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_STOP_SEQUENCES_KEY]).toEqual([ + "DONE", + "STOP", + ]); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_TEMPERATURE_KEY]).toBe(0.2); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_TOP_P_KEY]).toBe(0.8); + expect(attributes[OpenTelemetryConstants.GEN_AI_DATA_SOURCE_ID_KEY]).toBe("sharepoint"); + expect(attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_TYPE_KEY]).toBe("json"); + expect( + JSON.parse(attributes[OpenTelemetryConstants.GEN_AI_SYSTEM_INSTRUCTIONS_KEY] as string), + ).toEqual([{ type: "text", content: "Answer with JSON only." }]); + expect(attributes[OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY]).toEqual([ + "stop", + ]); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY]).toBe(120); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY]).toBe(48); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY]).toBe(12); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY]).toBe(3); + }); + + it("should record late response parameters without changing endpoint or message behavior", () => { + const details: InvokeAgentScopeDetails = { + endpoint: { host: "agent-api.contoso.com", port: 8443 }, + }; + const scope = InvokeAgentScope.start( + { ...testRequest, content: "Hello agent" }, + details, + testAgentDetails, + ); + + scope.recordResponse("Done"); + scope.recordResponseParameters({ + finishReasons: ["stop"], + inputTokens: 0, + outputTokens: 24, + cacheWriteInputTokens: 0, + cacheReadInputTokens: 2, + }); + scope.dispose(); + + const attributes = getLastSpan().attributes; + expect(attributes[OpenTelemetryConstants.SERVER_ADDRESS_KEY]).toBe("agent-api.contoso.com"); + expect(attributes[OpenTelemetryConstants.SERVER_PORT_KEY]).toBe(8443); + expect( + JSON.parse(attributes[OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY] as string)[0] + .parts[0].content, + ).toBe("Hello agent"); + expect( + JSON.parse(attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY] as string)[0] + .parts[0].content, + ).toBe("Done"); + expect(attributes[OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY]).toEqual([ + "stop", + ]); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY]).toBe(24); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY]).toBe(2); + }); + + it("should omit absent request and response parameters", () => { + const scope = InvokeAgentScope.start( + testRequest, + { + requestParameters: { + model: undefined, + stopSequences: undefined, + outputType: undefined, + }, + responseParameters: { + finishReasons: undefined, + inputTokens: undefined, + }, + }, + testAgentDetails, + ); + scope.dispose(); + + const attributes = getLastSpan().attributes; + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_MODEL_KEY]).toBeUndefined(); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_STOP_SEQUENCES_KEY]).toBeUndefined(); + expect(attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_TYPE_KEY]).toBeUndefined(); + expect(attributes[OpenTelemetryConstants.GEN_AI_RESPONSE_FINISH_REASONS_KEY]).toBeUndefined(); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY]).toBeUndefined(); + }); + + it("should emit zero-valued request and response numbers", () => { + const scope = InvokeAgentScope.start( + testRequest, + { + requestParameters: { + seed: 0, + choiceCount: 0, + frequencyPenalty: 0, + maxTokens: 0, + presencePenalty: 0, + temperature: 0, + topP: 0, + }, + responseParameters: { + inputTokens: 0, + outputTokens: 0, + cacheWriteInputTokens: 0, + cacheReadInputTokens: 0, + }, + }, + testAgentDetails, + ); + scope.dispose(); + + const attributes = getLastSpan().attributes; + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_SEED_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_CHOICE_COUNT_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_FREQUENCY_PENALTY_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_MAX_TOKENS_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_PRESENCE_PENALTY_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_TEMPERATURE_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_REQUEST_TOP_P_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_INPUT_TOKENS_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_OUTPUT_TOKENS_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_CACHE_WRITE_INPUT_TOKENS_KEY]).toBe(0); + expect(attributes[OpenTelemetryConstants.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_KEY]).toBe(0); + }); + + it("should emit a deterministic fallback for circular system instructions", () => { + const circularInstruction: SystemInstructionPart = { type: "custom" }; + circularInstruction.circular = circularInstruction; + + expect(() => { + const scope = InvokeAgentScope.start( + testRequest, + { + requestParameters: { + systemInstructions: [circularInstruction], + }, + }, + testAgentDetails, + ); + scope.dispose(); + }).not.toThrow(); + + const attributes = getLastSpan().attributes; + expect( + JSON.parse(attributes[OpenTelemetryConstants.GEN_AI_SYSTEM_INSTRUCTIONS_KEY] as string), + ).toEqual([ + { + type: "text", + content: "[serialization failed: 1 instruction]", + }, + ]); + }); + + it("should propagate the common agent provider name", () => { + const scope = InvokeAgentScope.start( + testRequest, + {}, + { ...testAgentDetails, providerName: "copilot" }, + ); + scope.dispose(); + + expect(getLastSpan().attributes[OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY]).toBe( + "copilot", + ); + }); + + it("should let inference details override the common agent provider name", () => { + const scope = InferenceScope.start( + testRequest, + { + operationName: InferenceOperationType.CHAT, + model: "gpt-4o", + providerName: "azure-openai", + }, + { ...testAgentDetails, providerName: "copilot" }, + ); + scope.dispose(); + + expect(getLastSpan().attributes[OpenTelemetryConstants.GEN_AI_PROVIDER_NAME_KEY]).toBe( + "azure-openai", + ); + }); + }); + describe("OutputScope", () => { it("should create scope with agent and request details", () => { const spy = vi.spyOn(OpenTelemetryScope.prototype as any, "setTagMaybe");