feat(batch-evaluation): add --output-config, rename --qualifier to --endpoint, require timestamp timezones

package.json

@@ -64,8 +64,8 @@
   "dependencies": {
     "@aws-cdk/toolkit-lib": "1.38.2",
     "@aws-sdk/client-bedrock-agent": "^3.1092.0",
-    "@aws-sdk/client-bedrock-agentcore": "^3.1092.0",
-    "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0",
+    "@aws-sdk/client-bedrock-agentcore": "^3.1129.0",
+    "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0",
     "@aws-sdk/client-cloudformation": "^3.1092.0",
     "@aws-sdk/client-cloudwatch-logs": "^3.1092.0",
     "@aws-sdk/client-iam": "^3.1080.0",

src/core/eval.tsx

@@ -679,6 +679,7 @@ export class EvalClient implements CoreEvalClient {
         dataSourceConfig,
         evaluationMetadata: input.groundTruth ? { sessionMetadata: input.groundTruth } : undefined,
         kmsKeyArn: input.kmsKeyArn,
+        outputConfig: input.outputConfig,
       }),
     );
   }

src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.2a1e0be72c84e19a.json

@@ -0,0 +1,21 @@
+{
+  "batchEvaluationId": "golden_batch_evaluate_outputconfig1-936a801ab2",
+  "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_evaluate_outputconfig1-936a801ab2",
+  "batchEvaluationName": "golden_batch_evaluate_outputconfig1",
+  "status": "PENDING",
+  "createdAt": {
+    "$date": "2026-09-11T20:52:41.450Z"
+  },
+  "evaluators": [
+    {
+      "evaluatorId": "Builtin.Helpfulness"
+    }
+  ],
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logStreamName": "run-golden_batch_evaluate_outputconfig1-936a801ab2",
+      "metricsNamespace": "Company/AgentEvaluations",
+      "resultDestination": "SOURCE_LOG_GROUP"
+    }
+  }
+}

src/handlers/eval/batch-evaluation/__fixtures__/StartBatchEvaluationCommand.374d4ae3a6654fca.json

@@ -0,0 +1,21 @@
+{
+  "batchEvaluationId": "golden_batch_simulate_endpoint1-120c119e26",
+  "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_simulate_endpoint1-120c119e26",
+  "batchEvaluationName": "golden_batch_simulate_endpoint1",
+  "status": "PENDING",
+  "createdAt": {
+    "$date": "2026-09-11T20:52:44.223Z"
+  },
+  "evaluators": [
+    {
+      "evaluatorId": "Builtin.Helpfulness"
+    }
+  ],
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logStreamName": "run-golden_batch_simulate_endpoint1-120c119e26",
+      "metricsNamespace": "Bedrock-AgentCore/Evaluations",
+      "resultDestination": "SOURCE_LOG_GROUP"
+    }
+  }
+}

src/handlers/eval/batch-evaluation/__fixtures__/evaluate-output-config.golden.json

@@ -0,0 +1,19 @@
+{
+  "batchEvaluationId": "golden_batch_evaluate_outputconfig1-936a801ab2",
+  "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_evaluate_outputconfig1-936a801ab2",
+  "batchEvaluationName": "golden_batch_evaluate_outputconfig1",
+  "status": "PENDING",
+  "createdAt": "2026-09-11T20:52:41.450Z",
+  "evaluators": [
+    {
+      "evaluatorId": "Builtin.Helpfulness"
+    }
+  ],
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logStreamName": "run-golden_batch_evaluate_outputconfig1-936a801ab2",
+      "metricsNamespace": "Company/AgentEvaluations",
+      "resultDestination": "SOURCE_LOG_GROUP"
+    }
+  }
+}

src/handlers/eval/batch-evaluation/__fixtures__/simulate-endpoint-output-config.golden.json

@@ -0,0 +1,13 @@
+{
+  "batchEvaluationId": "golden_batch_simulate_endpoint1-120c119e26",
+  "status": "PENDING",
+  "examplesInvoked": 1,
+  "examplesFailed": 0,
+  "sessions": [
+    {
+      "exampleId": "e1",
+      "sessionId": "00000000-0000-4000-8000-000000000001"
+    }
+  ],
+  "failures": []
+}

src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx

@@ -44,6 +44,8 @@ const FIXTURE_EVAL_AGENT = "asdf_MyAgent-3s5axvBC6Q";
 const FIXTURE_EVAL_NAME = "golden_batch_evaluate_fixture685";
 
 const FIXTURE_SIMULATE_NAME = "golden_batch_simulate_fixture1";
+const FIXTURE_OUTPUT_CONFIG_NAME = "golden_batch_evaluate_outputconfig1";
+const FIXTURE_SIMULATE_ENDPOINT_NAME = "golden_batch_simulate_endpoint1";
 const FIXTURE_SIMULATE_DATASET = join(FIXTURES, "simulate-ds.jsonl");
 
 function createFixtureCore(): CoreClient {
@@ -185,4 +187,72 @@ describe("eval batch-evaluation (fixture-backed)", () => {
 
     matchGolden(FIXTURES, "simulate.golden.json", io.stdout());
   }, 180_000);
+
+  test("evaluate submits a job with a customer-supplied output config", async () => {
+    const stdout = await run([
+      "eval",
+      "batch-evaluation",
+      "evaluate",
+      "--agent",
+      FIXTURE_EVAL_AGENT,
+      "--evaluators",
+      "Builtin.Helpfulness",
+      "--name",
+      FIXTURE_OUTPUT_CONFIG_NAME,
+      "--output-config",
+      '{"cloudWatchConfig":{"resultDestination":"SOURCE_LOG_GROUP","metricsNamespace":"Company/AgentEvaluations"}}',
+      "--json",
+    ]);
+
+    matchGolden(FIXTURES, "evaluate-output-config.golden.json", stdout);
+  });
+
+  test("simulate passes --endpoint through and submits with an output config", async () => {
+    let n = 0;
+    const { createControlClient, createDataClient, createIamClient, createLogsClient } =
+      fixtureFactories(FIXTURES);
+    const core = new CoreClient({
+      createControlClient,
+      createDataClient,
+      createIamClient,
+      createLogsClient,
+      logger: createSilentLogger(),
+      newSessionId: () => `00000000-0000-4000-8000-${String(++n).padStart(12, "0")}`,
+    });
+    const io = testIO();
+    const root = createRootHandler(core, {
+      io: io.io,
+      logger: createSilentLogger(),
+      globalConfigAccessor: new TestGlobalConfigAccessor(),
+    });
+
+    await root.route([
+      "node",
+      "agentcore",
+      "eval",
+      "batch-evaluation",
+      "simulate",
+      "--runtime-id",
+      FIXTURE_EVAL_AGENT,
+      "--endpoint",
+      "DEFAULT",
+      "--payload-template",
+      '{"prompt":"{input}"}',
+      "--dataset",
+      FIXTURE_SIMULATE_DATASET,
+      "--evaluators",
+      "Builtin.Helpfulness",
+      "--name",
+      FIXTURE_SIMULATE_ENDPOINT_NAME,
+      "--output-config",
+      '{"cloudWatchConfig":{"resultDestination":"SOURCE_LOG_GROUP"}}',
+      "--ingestion-wait-ms",
+      "0",
+      "--json",
+      "--region",
+      REGION,
+    ]);
+
+    matchGolden(FIXTURES, "simulate-endpoint-output-config.golden.json", io.stdout());
+  }, 180_000);
 });

src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx

@@ -34,10 +34,14 @@ const RESULTS: BatchEvaluationResultEntry[] = [
   { evaluatorId: "Builtin.Helpfulness", level: "Session", sessionId: "s1", score: 5 },
 ];
 
-async function run(args: string[], configure?: (core: TestCoreClient) => void) {
+async function run(
+  args: string[],
+  configure?: (core: TestCoreClient) => void,
+  ioOptions?: { stdin?: string },
+) {
   const core = new TestCoreClient();
   configure?.(core);
-  const io = testIO();
+  const io = testIO(ioOptions);
   const root = createRootHandler(core, {
     io: io.io,
     logger: createSilentLogger(),
@@ -262,3 +266,97 @@ describe("eval batch-evaluation simulate", () => {
     expect(JSON.parse(stdout).failures).toEqual([{ exampleId: "bad", error: "HTTP 500" }]);
   });
 });
+
+describe("eval batch-evaluation evaluate --output-config", () => {
+  const BASE = [
+    "eval",
+    "batch-evaluation",
+    "evaluate",
+    "--agent",
+    "r-1",
+    "--evaluators",
+    "Builtin.Helpfulness",
+    "--name",
+    "eval-1",
+  ];
+  function startInput(core: TestCoreClient) {
+    const call = core.eval.calls.find((c) => c.method === "startBatchEvaluation");
+    expect(call).toBeDefined();
+    return call!.args[0] as { outputConfig?: unknown };
+  }
+
+  test("is left undefined when omitted, so the service keeps its default destination", async () => {
+    const { core } = await run(BASE);
+    expect(startInput(core).outputConfig).toBeUndefined();
+  });
+
+  test("rejects malformed JSON before any SDK call", async () => {
+    const core = new TestCoreClient();
+    const io = testIO();
+    const root = createRootHandler(core, {
+      io: io.io,
+      logger: createSilentLogger(),
+      globalConfigAccessor: new TestGlobalConfigAccessor(),
+    });
+    await expect(
+      root.route([
+        "node",
+        "agentcore",
+        ...BASE,
+        "--output-config",
+        "{not json",
+        "--region",
+        "us-west-2",
+      ]),
+    ).rejects.toThrow(/Invalid JSON for option '--output-config'/);
+    expect(core.eval.calls).toEqual([]);
+  });
+});
+
+describe("eval batch-evaluation simulate --endpoint and --output-config", () => {
+  const BASE = [
+    "eval",
+    "batch-evaluation",
+    "simulate",
+    "--runtime-id",
+    "r-1",
+    "--payload-template",
+    '{"prompt":"{input}"}',
+    "--dataset",
+    "/tmp/ds.jsonl",
+    "--evaluators",
+    "Builtin.Helpfulness",
+    "--name",
+    "sim-1",
+  ];
+  const invoked = (c: TestCoreClient) =>
+    c.eval.setInvokeDatasetResponse({
+      sessions: [{ exampleId: "e1", sessionId: "s1" }],
+      invoked: 1,
+      failed: 0,
+      failures: [],
+    });
+
+  test("malformed --output-config aborts before any Runtime is invoked", async () => {
+    const core = new TestCoreClient();
+    invoked(core);
+    const io = testIO();
+    const root = createRootHandler(core, {
+      io: io.io,
+      logger: createSilentLogger(),
+      globalConfigAccessor: new TestGlobalConfigAccessor(),
+    });
+    await expect(
+      root.route([
+        "node",
+        "agentcore",
+        ...BASE,
+        "--output-config",
+        "{not json",
+        "--region",
+        "us-west-2",
+      ]),
+    ).rejects.toThrow(/Invalid JSON for option '--output-config'/);
+    expect(core.eval.calls.map((c) => c.method)).not.toContain("invokeDataset");
+  });
+});

src/handlers/eval/batch-evaluation/evaluate/index.tsx

@@ -7,6 +7,7 @@ import type { Core } from "../../../types";
 import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore";
 import { coreOptsFromCtx, parseJsonFlag } from "../../../utils";
 import { SessionSource } from "../../sessionSource";
+import { BatchOutputConfig } from "../outputConfig";
 
 const CONFIGURATION = "Configuration:";
 const EVALUATION = "Evaluation:";
@@ -59,6 +60,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
         z.string().optional(),
         { group: EVALUATION, help: groundTruthHelp },
       ),
+      ...BatchOutputConfig.flags,
     ],
     handle: async (ctx, flags) => {
       if (!flags["name"]) {
@@ -78,6 +80,8 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
         await resolver.resolveText("ground-truth", flags["ground-truth"]),
       );
 
+      const outputConfig = await BatchOutputConfig.resolve(flags["output-config"], io);
+
       const response = await core.eval.startBatchEvaluation(
         {
           name: flags["name"],
@@ -86,6 +90,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
           source,
           groundTruth,
           kmsKeyArn: flags["kms-key-arn"],
+          outputConfig,
         },
         coreOptsFromCtx(ctx),
       );

src/handlers/eval/batch-evaluation/outputConfig.tsx

@@ -0,0 +1,54 @@
+import type { OutputConfig } from "@aws-sdk/client-bedrock-agentcore";
+import z from "zod";
+import { SourceResolver, type AppIO } from "../../../io";
+import { flag } from "../../../router";
+import { parseJsonFlag } from "../../utils";
+
+const outputConfigHelp = `(JSON: tagged union object)
+Where evaluation results and metrics are written. Omit it and results go to the
+service-managed default location. Only top-level key: cloudWatchConfig.
+
+Accepts inline JSON, file://<path>, or - to read stdin.
+
+JSON syntax:
+  {
+    "cloudWatchConfig": {
+      "logGroupName": "string",      // result log group; omit for
+                                     // SOURCE_LOG_GROUP, and it cannot sit
+                                     // under /aws/bedrock-agentcore/evaluations/
+      "logStreamName": "string",     // result log stream
+      "metricsNamespace": "string",  // defaults to
+                                     // Bedrock-AgentCore/Evaluations; cannot
+                                     // begin with "AWS/"
+      "resultDestination": "DEDICATED_LOG_GROUP" | "SOURCE_LOG_GROUP"
+                                     // DEDICATED_LOG_GROUP (default) writes to
+                                     // a dedicated result group;
+                                     // SOURCE_LOG_GROUP writes back to the log
+                                     // group the traces were read from
+    }
+  }
+
+API reference:
+  https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_OutputConfig.html
+
+Example:
+  --output-config '{"cloudWatchConfig":{"logGroupName":"/company/agent-evaluations","metricsNamespace":"Company/AgentEvaluations","resultDestination":"DEDICATED_LOG_GROUP"}}'`;
+
+export class BatchOutputConfig {
+  static readonly flags = [
+    flag(
+      "output-config",
+      "where results and metrics are written (JSON OutputConfig)",
+      z.string().optional(),
+      { group: "Result output:", help: outputConfigHelp },
+    ),
+  ] as const;
+
+  static async resolve(value: string | undefined, io: AppIO): Promise<OutputConfig | undefined> {
+    const resolver = new SourceResolver({ stdin: io.stdin });
+    return parseJsonFlag<OutputConfig>(
+      "output-config",
+      await resolver.resolveText("output-config", value),
+    );
+  }
+}