feat(online-eval): add --output-config and --tags, --description on update, and widen the managed role

package.json

@@ -64,8 +64,8 @@
   "dependencies": {
     "@aws-cdk/toolkit-lib": "1.38.2",
     "@aws-sdk/client-bedrock-agent": "^3.1092.0",
-    "@aws-sdk/client-bedrock-agentcore": "^3.1092.0",
-    "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0",
+    "@aws-sdk/client-bedrock-agentcore": "^3.1129.0",
+    "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0",
     "@aws-sdk/client-cloudformation": "^3.1092.0",
     "@aws-sdk/client-cloudwatch-logs": "^3.1092.0",
     "@aws-sdk/client-iam": "^3.1080.0",

src/core/eval.test.ts

@@ -0,0 +1,77 @@
+import { describe, expect, test } from "bun:test";
+import {
+  StartBatchEvaluationCommand,
+  type BedrockAgentCoreClient,
+} from "@aws-sdk/client-bedrock-agentcore";
+import type { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control";
+import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs";
+import type { IAMClient } from "@aws-sdk/client-iam";
+import { CoreClient } from "./index";
+import { createSilentLogger } from "../testing";
+import type { OutputConfig } from "@aws-sdk/client-bedrock-agentcore";
+
+const OPTIONS = { region: "us-west-2" };
+
+const RAW_SOURCE = {
+  origin: "raw" as const,
+  dataSourceConfig: {
+    cloudWatchLogs: { serviceNames: ["my_agent.DEFAULT"], logGroupNames: ["/some/group"] },
+  },
+};
+
+function coreWithCapturedDataCommands() {
+  const sent: unknown[] = [];
+  const unusable = (name: string) => () => {
+    throw new Error(`this test should not construct the ${name} client`);
+  };
+  const core = new CoreClient({
+    createDataClient: () =>
+      ({
+        send: async (command: unknown) => {
+          sent.push(command);
+          return { batchEvaluationId: "batch-eval-1", status: "IN_PROGRESS" };
+        },
+      }) as unknown as BedrockAgentCoreClient,
+    createControlClient: unusable("control") as unknown as () => BedrockAgentCoreControlClient,
+    createIamClient: unusable("IAM") as unknown as () => IAMClient,
+    createLogsClient: unusable("logs") as unknown as () => CloudWatchLogsClient,
+    logger: createSilentLogger(),
+  });
+  const startInput = () => {
+    const command = sent.find((c) => c instanceof StartBatchEvaluationCommand);
+    expect(command).toBeDefined();
+    return (command as StartBatchEvaluationCommand).input;
+  };
+  return { core, startInput };
+}
+
+describe("CoreClient.eval.startBatchEvaluation", () => {
+  test("forwards outputConfig to the request unchanged", async () => {
+    const outputConfig = {
+      cloudWatchConfig: {
+        logGroupName: "/company/agent-evaluations",
+        metricsNamespace: "Company/AgentEvaluations",
+        resultDestination: "DEDICATED_LOG_GROUP",
+      },
+    } as OutputConfig;
+    const { core, startInput } = coreWithCapturedDataCommands();
+
+    await core.eval.startBatchEvaluation(
+      { name: "eval-1", evaluatorIds: ["Builtin.Helpfulness"], source: RAW_SOURCE, outputConfig },
+      OPTIONS,
+    );
+
+    expect(startInput().outputConfig).toEqual(outputConfig);
+  });
+
+  test("omits outputConfig when the caller did not supply one", async () => {
+    const { core, startInput } = coreWithCapturedDataCommands();
+
+    await core.eval.startBatchEvaluation(
+      { name: "eval-1", evaluatorIds: ["Builtin.Helpfulness"], source: RAW_SOURCE },
+      OPTIONS,
+    );
+
+    expect(startInput().outputConfig).toBeUndefined();
+  });
+});

src/core/eval.tsx

@@ -123,6 +123,8 @@ import type {
   DatasetUpdateProgressEvent,
   DatasetUpdateResult,
   RoleScopeWarning,
+  RoleScopeKind,
+  OnlineEvalOutputConfig,
   CoreEvalClient,
   CreateConfigurationBundleInput,
   CreateConfigBasedABTestInput,
@@ -679,6 +681,7 @@ export class EvalClient implements CoreEvalClient {
         dataSourceConfig,
         evaluationMetadata: input.groundTruth ? { sessionMetadata: input.groundTruth } : undefined,
         kmsKeyArn: input.kmsKeyArn,
+        outputConfig: input.outputConfig,
       }),
     );
   }
@@ -988,6 +991,9 @@ export class EvalClient implements CoreEvalClient {
           options.region,
           logGroupNamesOf(dataSourceConfig),
           await evaluatorKmsKeys(input.evaluatorIds ?? [], control),
+          // Read only to widen the write scope to the chosen destination; the
+          // request object below still gets the caller's object untouched.
+          { outputConfig: input.outputConfig },
         )
       ).roleArn;
 
@@ -997,8 +1003,10 @@ export class EvalClient implements CoreEvalClient {
       rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters),
       dataSourceConfig,
       evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })),
+      outputConfig: input.outputConfig,
       evaluationExecutionRoleArn,
       enableOnCreate: input.enableOnCreate ?? true,
+      tags: input.tags,
     });
 
     // A role provisioned moments ago may not be assumable yet (IAM is eventually
@@ -1236,6 +1244,11 @@ export class EvalClient implements CoreEvalClient {
         ? dataSourceConfig
         : undefined;
 
+    const outputMoved = update.outputConfig !== undefined;
+    const effectiveOutputConfig = update.outputConfig ?? current.outputConfig;
+    const scopeKind: RoleScopeKind =
+      movedTo !== undefined && outputMoved ? "input-and-output" : outputMoved ? "output" : "input";
+
     const configName = current.onlineEvaluationConfigName;
     const roleArn = update.evaluationExecutionRoleArn ?? current.evaluationExecutionRoleArn;
     const managedRoleName =
@@ -1245,26 +1258,33 @@ export class EvalClient implements CoreEvalClient {
       isManagedOnlineEvalRole(roleArn, configName)
         ? configName
         : undefined;
-    const refreshManagedRole = movedTo !== undefined && managedRoleName !== undefined;
-
-    if (movedTo !== undefined && managedRoleName === undefined && roleArn) {
+    const scopeChanged = movedTo !== undefined || outputMoved;
+    const refreshManagedRole = scopeChanged && managedRoleName !== undefined;
+    const affectedLogGroups = [
+      ...(movedTo !== undefined ? logGroupNamesOf(movedTo) : []),
+      ...(outputMoved ? destinationLogGroupNames(update.outputConfig, dataSourceConfig) : []),
+    ];
+
+    if (scopeChanged && managedRoleName === undefined && roleArn) {
       roleScopeWarning = {
         reason: "custom-role",
         roleArn,
-        logGroupNames: logGroupNamesOf(movedTo),
+        scope: scopeKind,
+        logGroupNames: affectedLogGroups,
       };
-    } else if (movedTo !== undefined && !refreshManagedRole && roleArn) {
+    } else if (scopeChanged && !refreshManagedRole && roleArn) {
       // managed role, but the caller declined the refresh
       roleScopeWarning = {
         reason: "update-declined",
         roleArn,
-        logGroupNames: logGroupNamesOf(movedTo),
+        scope: scopeKind,
+        logGroupNames: affectedLogGroups,
       };
     }
 
     if (refreshManagedRole && update.updateRole !== false) {
       const iam = this.clients.iam({ region: options.region });
-      const newLogGroups = logGroupNamesOf(movedTo);
+      const newLogGroups = dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : [];
       const oldLogGroups = current.dataSourceConfig
         ? logGroupNamesOf(current.dataSourceConfig)
         : [];
@@ -1289,23 +1309,26 @@ export class EvalClient implements CoreEvalClient {
         options.region,
         newLogGroups,
         kmsKeys,
-        resourceNameFromArn(roleArn!),
+        { roleName: resourceNameFromArn(roleArn!), outputConfig: effectiveOutputConfig },
       );
       const oldPolicyName = scopePolicyName(
         executionPolicy(
           options.region,
           accountIdFromRoleArn(managedRoleArn),
           oldLogGroups,
           kmsKeys,
+          current.outputConfig,
         ),
       );
 
       const response = await control.send(
         new UpdateOnlineEvaluationConfigCommand({
           onlineEvaluationConfigId: id,
+          description: update.description,
           rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters),
           dataSourceConfig,
           evaluators,
+          outputConfig: update.outputConfig,
         }),
       );
 
@@ -1322,6 +1345,7 @@ export class EvalClient implements CoreEvalClient {
           roleScopeWarning = {
             reason: "stale-scope",
             roleArn: roleArn!,
+            scope: scopeKind,
             logGroupNames: oldLogGroups,
           };
         }
@@ -1332,9 +1356,11 @@ export class EvalClient implements CoreEvalClient {
     const response = await control.send(
       new UpdateOnlineEvaluationConfigCommand({
         onlineEvaluationConfigId: id,
+        description: update.description,
         rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters),
         dataSourceConfig,
         evaluators,
+        outputConfig: update.outputConfig,
         evaluationExecutionRoleArn: update.evaluationExecutionRoleArn,
       }),
     );
@@ -2200,6 +2226,18 @@ function logGroupNamesOf(dataSourceConfig: DataSourceConfig): string[] {
     : [];
 }
 
+function destinationLogGroupNames(
+  outputConfig: OnlineEvalOutputConfig | undefined,
+  dataSourceConfig: DataSourceConfig | undefined,
+): string[] {
+  const cloudWatch = outputConfig?.cloudWatchConfig;
+  if (!cloudWatch) return [];
+  if (cloudWatch.resultDestination === "SOURCE_LOG_GROUP") {
+    return dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : [];
+  }
+  return cloudWatch.logGroupName ? [cloudWatch.logGroupName] : [];
+}
+
 // runtimeIdFromLogGroup recovers the runtime id embedded in a log group path
 // produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for a
 // new --endpoint without the caller passing --agent again. Returns undefined for

src/core/onlineEvalExecutionRole.test.ts

@@ -110,3 +110,68 @@ test("gives identical policies the same name", () => {
     scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])),
   );
 });
+
+function writeStatement(policy: string) {
+  return statements(policy).find((s) => s.Sid === "WriteEvaluationResults");
+}
+
+const SERVICE_RESULTS = `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/evaluations/*`;
+
+test("a config with no output destination keeps the service namespace as a bare string", () => {
+  const write = writeStatement(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []));
+
+  expect(write?.Resource).toBe(SERVICE_RESULTS);
+  expect(Array.isArray(write?.Resource)).toBe(false);
+});
+
+test("a customer-named dedicated group is granted alongside the service namespace", () => {
+  const write = writeStatement(
+    executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+      cloudWatchConfig: {
+        logGroupName: "/company/agent-evaluations",
+        resultDestination: "DEDICATED_LOG_GROUP",
+      },
+    }),
+  );
+
+  expect(write?.Resource).toEqual([
+    SERVICE_RESULTS,
+    `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/company/agent-evaluations*`,
+  ]);
+  expect(write?.Action).toContain("logs:CreateLogGroup");
+});
+
+test("SOURCE_LOG_GROUP grants writes to the groups the traces are read from", () => {
+  const write = writeStatement(
+    executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+      cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" },
+    }),
+  );
+
+  expect(write?.Resource).toEqual([
+    SERVICE_RESULTS,
+    `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`,
+  ]);
+});
+
+test("a destination already inside the service namespace adds nothing", () => {
+  const write = writeStatement(
+    executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+      cloudWatchConfig: {
+        logGroupName: "/aws/bedrock-agentcore/evaluations/online-evaluations/results/default",
+        resultDestination: "DEDICATED_LOG_GROUP",
+      },
+    }),
+  );
+
+  expect(write?.Resource).toBe(SERVICE_RESULTS);
+});
+
+test("changing the destination changes the policy name, so a re-scope is a new grant", () => {
+  const before = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []);
+  const after = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], {
+    cloudWatchConfig: { logGroupName: "/company/agent-evaluations" },
+  });
+
+  expect(scopePolicyName(after)).not.toBe(scopePolicyName(before));
+});