From 7a95dc7bf0761cf97ec4a0952bb898677f19a617 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 10 Sep 2026 23:23:21 +0000 Subject: [PATCH 1/2] feat(batch-evaluation): add --output-config and rename --qualifier to --endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --output-config lets a customer say where results and metrics are written instead of taking the service-managed default. An API-shaped passthrough: BatchOutputConfig resolves inline JSON, file://, or stdin and hands the parsed object to StartBatchEvaluation with field names and nested values untouched. Field documentation goes in the flag's `help:` block, rendered under "Parameter details". `simulate --qualifier` becomes `--endpoint`, matching SessionSource, which has always called the same concept --endpoint. The InvokeDatasetInput field stays `qualifier` — that is the Runtime API's name, not ours. simulate resolves --output-config before invoking the Runtime, since the replay bills the customer per dataset example and malformed JSON must not surface only after the whole run. Dependencies move to @aws-sdk/client-bedrock-agentcore 3.1129.0, the first release exposing request-side outputConfig on StartBatchEvaluationRequest. The generated type is used directly; no cast papers over an older model. src/core/eval.test.ts covers the seam the other suites miss: the TestCoreClient suites assert what a handler hands to Core, and the fixture suites need an account to re-record. Deleting Core's outputConfig forwarding passed every test until this one existed. Timestamp handling is unchanged. An earlier revision required an explicit timezone on --start-time/--end-time; that is a breaking change and has been dropped. --- bun.lock | 60 +++++++- package.json | 4 +- src/core/eval.test.ts | 77 ++++++++++ src/core/eval.tsx | 1 + .../batch-evaluation.test.tsx | 144 +++++++++++++++++- .../eval/batch-evaluation/evaluate/index.tsx | 5 + .../eval/batch-evaluation/outputConfig.tsx | 54 +++++++ .../eval/batch-evaluation/simulate/index.tsx | 13 +- src/handlers/eval/types.tsx | 2 + 9 files changed, 348 insertions(+), 12 deletions(-) create mode 100644 src/core/eval.test.ts create mode 100644 src/handlers/eval/batch-evaluation/outputConfig.tsx diff --git a/bun.lock b/bun.lock index a99d76eea..b254f4aa7 100644 --- a/bun.lock +++ b/bun.lock @@ -7,8 +7,8 @@ "dependencies": { "@aws-cdk/toolkit-lib": "1.38.2", "@aws-sdk/client-bedrock-agent": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0", + "@aws-sdk/client-bedrock-agentcore": "^3.1129.0", + "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0", "@aws-sdk/client-cloudformation": "^3.1092.0", "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", "@aws-sdk/client-iam": "^3.1080.0", @@ -95,9 +95,9 @@ "@aws-sdk/client-bedrock-agent": ["@aws-sdk/client-bedrock-agent@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-RAjn1g6X+u4WwnuwxYU8sKna5q1waTBCM+kzXWAvxSaNbgRsXFL6ZylPjOl0qCDZ7x6aVTj7Jb7Q0z6DZifP3w=="], - "@aws-sdk/client-bedrock-agentcore": ["@aws-sdk/client-bedrock-agentcore@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-VxagAO73X0efH+5oQfOqWc2/q1wTkiZ1IkBg7L0GnPBo4NFYw37H8klNbbrTqoluMAzRiBwsHoGX2jfnjZlDOQ=="], + "@aws-sdk/client-bedrock-agentcore": ["@aws-sdk/client-bedrock-agentcore@3.1130.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-node": "^3.972.83", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-HgHR24kDJwRN9NNrtWNeuyrJZgl0nXOikeeD2HBAprsRXFl3tvZQw6hHOADHR41sjT4z3+SmqIqN5U7JiaF40Q=="], - "@aws-sdk/client-bedrock-agentcore-control": ["@aws-sdk/client-bedrock-agentcore-control@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-zta1MHik3+WbRUVwRFyFWSGTEfhaMPDDG+qdzmLAAyK3m7gqYHJr6+p+6dvBeiO6U15FQ1PWSjj2C3O5ikiyAw=="], + "@aws-sdk/client-bedrock-agentcore-control": ["@aws-sdk/client-bedrock-agentcore-control@3.1130.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-node": "^3.972.83", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-EB3zB2zmxVsx+ZqusNjwBHhsqzWlkMLwyO7TIvc9tWHh02eV0lkmRT3HE4L6+DvZrE9R2x6KBl4OSkfU1TZvGA=="], "@aws-sdk/client-cloudcontrol": ["@aws-sdk/client-cloudcontrol@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-kpR+c528XFWFT04APYPKrhSCyzdEJPgxryR0wWs2dfIj7Ovqwak+1zLwFOS8zpNqDPEx8xn7j3dqA/HTCo+qLw=="], @@ -1531,6 +1531,14 @@ "@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="], + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/core": ["@aws-sdk/core@3.978.0", "", { "dependencies": { "@aws-sdk/types": "^3.974.5", "@aws-sdk/xml-builder": "^3.972.40", "@aws/lambda-invoke-store": "^0.3.0", "@smithy/core": "^3.33.3", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.17.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-2yX9LUmxPklVjSGTb8dfnWRJSiFQ3TeH2nn7G1mdKHTfnabzF0+gfrS8rYfLWmZrQ8A3mEcxMJjRc51dL5KWaA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.83", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-ini": "^3.973.16", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-jdso7ejzfRnatxMUZK4S/U6KbaDPCvfIV4XL+IQAPFDBt5rj5Fq595euqlK8Le4lNCMFR9oUpt+1l0aMgaayOQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/core": ["@aws-sdk/core@3.978.0", "", { "dependencies": { "@aws-sdk/types": "^3.974.5", "@aws-sdk/xml-builder": "^3.972.40", "@aws/lambda-invoke-store": "^0.3.0", "@smithy/core": "^3.33.3", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.17.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-2yX9LUmxPklVjSGTb8dfnWRJSiFQ3TeH2nn7G1mdKHTfnabzF0+gfrS8rYfLWmZrQ8A3mEcxMJjRc51dL5KWaA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.83", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-ini": "^3.973.16", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-jdso7ejzfRnatxMUZK4S/U6KbaDPCvfIV4XL+IQAPFDBt5rj5Fq595euqlK8Le4lNCMFR9oUpt+1l0aMgaayOQ=="], + "@aws-sdk/protocol-http/@smithy/protocol-http": ["@smithy/protocol-http@1.2.0", "", { "dependencies": { "@smithy/types": "^1.2.0", "tslib": "^2.5.0" } }, "sha512-GfGfruksi3nXdFok5RhgtOnWe5f6BndzYfmEXISD+5gAGdayFGpjWu5pIqIweTudMtse20bGbc+7MFZXT1Tb8Q=="], "@aws-sdk/signature-v4/@smithy/signature-v4": ["@smithy/signature-v4@1.1.0", "", { "dependencies": { "@smithy/eventstream-codec": "^1.1.0", "@smithy/is-array-buffer": "^1.1.0", "@smithy/types": "^1.2.0", "@smithy/util-hex-encoding": "^1.1.0", "@smithy/util-middleware": "^1.1.0", "@smithy/util-uri-escape": "^1.1.0", "@smithy/util-utf8": "^1.1.0", "tslib": "^2.5.0" } }, "sha512-fDo3m7YqXBs7neciOePPd/X9LPm5QLlDMdIC4m1H6dgNLnXfLMFNIxEfPyohGA8VW9Wn4X8lygnPSGxDZSmp0Q=="], @@ -1651,6 +1659,30 @@ "@aws-cdk/cx-api/@aws-cdk/cloud-assembly-schema/semver": ["semver@7.8.5", "", { "bundled": true, "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-JN+JHruYZw3GUZB8YGAlDk4wTDPOEAEEdEzj5nS0xodWR4smzHsN7PnK2j6IeOsDIj2aqua5DSbhXl9Gtf90FQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.73", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uyYYnJOnlis8uQzaYGPd7N1JoioCoNpXgnkXYixsWJXHXgXyYi8WXJSDfofxJeWfQIGWLe2Nwyq60Uc7MZdVOg=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.973.16", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-login": "^3.972.78", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-i++ly+0Uxa+u3ebSSyr0S/3CFhFJDxCXT3+Zj+mW2bXenEx5bKGCdTIKFu39SgXBNhWDjex/8cXUx9MUTMCrTw=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-lYmXJa4gvq4xN1lrT5NiP5vIYYKcGWAdj8y+8o6dlcateB5eF3Dn8DtmjjHKfMBrTPAMr2pebIiX/UOj8c1/UA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.973.15", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/token-providers": "3.1129.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-6Jhcf4v0pSFdjk1EW2kvzuEBKD+UZ2uNcHUIglKKLndD20YhvkL2kdmDOV5/j4mYuWWwe/a1FQ1aomU86/Cg5Q=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.77", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uylIQSUWpfLuH2LovxEEfwzJGM/SabLOfLMg6YXu/E8jJEKUdpdILCVCQCdFvHyu/7dLJOHPMfrSwduxO56NkQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-JN+JHruYZw3GUZB8YGAlDk4wTDPOEAEEdEzj5nS0xodWR4smzHsN7PnK2j6IeOsDIj2aqua5DSbhXl9Gtf90FQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.73", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uyYYnJOnlis8uQzaYGPd7N1JoioCoNpXgnkXYixsWJXHXgXyYi8WXJSDfofxJeWfQIGWLe2Nwyq60Uc7MZdVOg=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.973.16", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-login": "^3.972.78", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-i++ly+0Uxa+u3ebSSyr0S/3CFhFJDxCXT3+Zj+mW2bXenEx5bKGCdTIKFu39SgXBNhWDjex/8cXUx9MUTMCrTw=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-lYmXJa4gvq4xN1lrT5NiP5vIYYKcGWAdj8y+8o6dlcateB5eF3Dn8DtmjjHKfMBrTPAMr2pebIiX/UOj8c1/UA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.973.15", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/token-providers": "3.1129.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-6Jhcf4v0pSFdjk1EW2kvzuEBKD+UZ2uNcHUIglKKLndD20YhvkL2kdmDOV5/j4mYuWWwe/a1FQ1aomU86/Cg5Q=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.77", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uylIQSUWpfLuH2LovxEEfwzJGM/SabLOfLMg6YXu/E8jJEKUdpdILCVCQCdFvHyu/7dLJOHPMfrSwduxO56NkQ=="], + "@aws-sdk/protocol-http/@smithy/protocol-http/@smithy/types": ["@smithy/types@1.2.0", "", { "dependencies": { "tslib": "^2.5.0" } }, "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA=="], "@aws-sdk/signature-v4/@smithy/signature-v4/@smithy/types": ["@smithy/types@1.2.0", "", { "dependencies": { "tslib": "^2.5.0" } }, "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA=="], @@ -1691,6 +1723,26 @@ "@aws-cdk/cloudformation-diff/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.78", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-eUtswnXu0+Ii9ieRK+0L7aPFV3Z/dnW2VntJzjBP9xs8s+8p5nBNuymIXtXwZ+5r5+XJP3e32nMkuZ/r0HozEA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1129.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-Sbl3rpzQdsG4ZK2zh0JWUYyZPKKorJlVOddA2T0DVbKJFrsW8J6wgnslxxUH04+WaBMr4A1HzJZvZX0xUvkniA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.78", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-eUtswnXu0+Ii9ieRK+0L7aPFV3Z/dnW2VntJzjBP9xs8s+8p5nBNuymIXtXwZ+5r5+XJP3e32nMkuZ/r0HozEA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1129.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-Sbl3rpzQdsG4ZK2zh0JWUYyZPKKorJlVOddA2T0DVbKJFrsW8J6wgnslxxUH04+WaBMr4A1HzJZvZX0xUvkniA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + "@aws-sdk/signature-v4/@smithy/signature-v4/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@1.1.0", "", { "dependencies": { "@smithy/is-array-buffer": "^1.1.0", "tslib": "^2.5.0" } }, "sha512-9m6NXE0ww+ra5HKHCHig20T+FAwxBAm7DIdwc/767uGWbRcY720ybgPacQNB96JMOI7xVr/CDa3oMzKmW4a+kw=="], "@typescript-eslint/typescript-estree/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], diff --git a/package.json b/package.json index 5ab4d027e..c2b2792cf 100644 --- a/package.json +++ b/package.json @@ -64,8 +64,8 @@ "dependencies": { "@aws-cdk/toolkit-lib": "1.38.2", "@aws-sdk/client-bedrock-agent": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0", + "@aws-sdk/client-bedrock-agentcore": "^3.1129.0", + "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0", "@aws-sdk/client-cloudformation": "^3.1092.0", "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", "@aws-sdk/client-iam": "^3.1080.0", diff --git a/src/core/eval.test.ts b/src/core/eval.test.ts new file mode 100644 index 000000000..8917393e0 --- /dev/null +++ b/src/core/eval.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from "bun:test"; +import { + StartBatchEvaluationCommand, + type BedrockAgentCoreClient, +} from "@aws-sdk/client-bedrock-agentcore"; +import type { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import type { IAMClient } from "@aws-sdk/client-iam"; +import { CoreClient } from "./index"; +import { createSilentLogger } from "../testing"; +import type { OutputConfig } from "@aws-sdk/client-bedrock-agentcore"; + +const OPTIONS = { region: "us-west-2" }; + +const RAW_SOURCE = { + origin: "raw" as const, + dataSourceConfig: { + cloudWatchLogs: { serviceNames: ["my_agent.DEFAULT"], logGroupNames: ["/some/group"] }, + }, +}; + +function coreWithCapturedDataCommands() { + const sent: unknown[] = []; + const unusable = (name: string) => () => { + throw new Error(`this test should not construct the ${name} client`); + }; + const core = new CoreClient({ + createDataClient: () => + ({ + send: async (command: unknown) => { + sent.push(command); + return { batchEvaluationId: "batch-eval-1", status: "IN_PROGRESS" }; + }, + }) as unknown as BedrockAgentCoreClient, + createControlClient: unusable("control") as unknown as () => BedrockAgentCoreControlClient, + createIamClient: unusable("IAM") as unknown as () => IAMClient, + createLogsClient: unusable("logs") as unknown as () => CloudWatchLogsClient, + logger: createSilentLogger(), + }); + const startInput = () => { + const command = sent.find((c) => c instanceof StartBatchEvaluationCommand); + expect(command).toBeDefined(); + return (command as StartBatchEvaluationCommand).input; + }; + return { core, startInput }; +} + +describe("CoreClient.eval.startBatchEvaluation", () => { + test("forwards outputConfig to the request unchanged", async () => { + const outputConfig = { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + metricsNamespace: "Company/AgentEvaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + } as OutputConfig; + const { core, startInput } = coreWithCapturedDataCommands(); + + await core.eval.startBatchEvaluation( + { name: "eval-1", evaluatorIds: ["Builtin.Helpfulness"], source: RAW_SOURCE, outputConfig }, + OPTIONS, + ); + + expect(startInput().outputConfig).toEqual(outputConfig); + }); + + test("omits outputConfig when the caller did not supply one", async () => { + const { core, startInput } = coreWithCapturedDataCommands(); + + await core.eval.startBatchEvaluation( + { name: "eval-1", evaluatorIds: ["Builtin.Helpfulness"], source: RAW_SOURCE }, + OPTIONS, + ); + + expect(startInput().outputConfig).toBeUndefined(); + }); +}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx index bf9080d51..61bd76802 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -679,6 +679,7 @@ export class EvalClient implements CoreEvalClient { dataSourceConfig, evaluationMetadata: input.groundTruth ? { sessionMetadata: input.groundTruth } : undefined, kmsKeyArn: input.kmsKeyArn, + outputConfig: input.outputConfig, }), ); } diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index 98d49f7b2..9170a30b5 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -34,10 +34,14 @@ const RESULTS: BatchEvaluationResultEntry[] = [ { evaluatorId: "Builtin.Helpfulness", level: "Session", sessionId: "s1", score: 5 }, ]; -async function run(args: string[], configure?: (core: TestCoreClient) => void) { +async function run( + args: string[], + configure?: (core: TestCoreClient) => void, + ioOptions?: { stdin?: string }, +) { const core = new TestCoreClient(); configure?.(core); - const io = testIO(); + const io = testIO(ioOptions); const root = createRootHandler(core, { io: io.io, logger: createSilentLogger(), @@ -262,3 +266,139 @@ describe("eval batch-evaluation simulate", () => { expect(JSON.parse(stdout).failures).toEqual([{ exampleId: "bad", error: "HTTP 500" }]); }); }); + +describe("eval batch-evaluation evaluate --output-config", () => { + const BASE = [ + "eval", + "batch-evaluation", + "evaluate", + "--agent", + "r-1", + "--evaluators", + "Builtin.Helpfulness", + "--name", + "eval-1", + ]; + const CONFIG = { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + metricsNamespace: "Company/AgentEvaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }; + + function startInput(core: TestCoreClient) { + const call = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect(call).toBeDefined(); + return call!.args[0] as { outputConfig?: unknown }; + } + + test("reaches the request unchanged from inline JSON", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(CONFIG)]); + expect(startInput(core).outputConfig).toEqual(CONFIG); + }); + + test("reaches the request unchanged from a file", async () => { + const path = `${process.env["TMPDIR"] ?? "/tmp"}/agentcore-output-config-${Bun.hash(JSON.stringify(CONFIG))}.json`; + await Bun.write(path, JSON.stringify(CONFIG)); + const { core } = await run([...BASE, "--output-config", `file://${path}`]); + expect(startInput(core).outputConfig).toEqual(CONFIG); + }); + + test("reaches the request unchanged from stdin", async () => { + const { core } = await run([...BASE, "--output-config", "-"], undefined, { + stdin: JSON.stringify(CONFIG), + }); + expect(startInput(core).outputConfig).toEqual(CONFIG); + }); + + test("is left undefined when omitted, so the service keeps its default destination", async () => { + const { core } = await run(BASE); + expect(startInput(core).outputConfig).toBeUndefined(); + }); + + test("rejects malformed JSON before any SDK call", async () => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route([ + "node", + "agentcore", + ...BASE, + "--output-config", + "{not json", + "--region", + "us-west-2", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + expect(core.eval.calls).toEqual([]); + }); +}); + +describe("eval batch-evaluation simulate --endpoint and --output-config", () => { + const BASE = [ + "eval", + "batch-evaluation", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluators", + "Builtin.Helpfulness", + "--name", + "sim-1", + ]; + const invoked = (c: TestCoreClient) => + c.eval.setInvokeDatasetResponse({ + sessions: [{ exampleId: "e1", sessionId: "s1" }], + invoked: 1, + failed: 0, + failures: [], + }); + + test("--endpoint drives both the Runtime invocation and the graded session source", async () => { + const { core } = await run([...BASE, "--endpoint", "BETA"], invoked); + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect((invoke!.args[0] as { qualifier?: string }).qualifier).toBe("BETA"); + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect((start!.args[0] as { source: { endpoint?: string } }).source.endpoint).toBe("BETA"); + }); + + test("malformed --output-config aborts before any Runtime is invoked", async () => { + const core = new TestCoreClient(); + invoked(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route([ + "node", + "agentcore", + ...BASE, + "--output-config", + "{not json", + "--region", + "us-west-2", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + expect(core.eval.calls.map((c) => c.method)).not.toContain("invokeDataset"); + }); + + test("a valid --output-config reaches the graded job", async () => { + const config = { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }; + const { core } = await run([...BASE, "--output-config", JSON.stringify(config)], invoked); + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect((start!.args[0] as { outputConfig?: unknown }).outputConfig).toEqual(config); + }); +}); diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index 8b95678ad..17e9d92f8 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -7,6 +7,7 @@ import type { Core } from "../../../types"; import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +import { BatchOutputConfig } from "../outputConfig"; const CONFIGURATION = "Configuration:"; const EVALUATION = "Evaluation:"; @@ -59,6 +60,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => z.string().optional(), { group: EVALUATION, help: groundTruthHelp }, ), + ...BatchOutputConfig.flags, ], handle: async (ctx, flags) => { if (!flags["name"]) { @@ -78,6 +80,8 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => await resolver.resolveText("ground-truth", flags["ground-truth"]), ); + const outputConfig = await BatchOutputConfig.resolve(flags["output-config"], io); + const response = await core.eval.startBatchEvaluation( { name: flags["name"], @@ -86,6 +90,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => source, groundTruth, kmsKeyArn: flags["kms-key-arn"], + outputConfig, }, coreOptsFromCtx(ctx), ); diff --git a/src/handlers/eval/batch-evaluation/outputConfig.tsx b/src/handlers/eval/batch-evaluation/outputConfig.tsx new file mode 100644 index 000000000..d3fead6b1 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/outputConfig.tsx @@ -0,0 +1,54 @@ +import type { OutputConfig } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { SourceResolver, type AppIO } from "../../../io"; +import { flag } from "../../../router"; +import { parseJsonFlag } from "../../utils"; + +const outputConfigHelp = `(JSON: tagged union object) +Where evaluation results and metrics are written. Omit it and results go to the +service-managed default location. Only top-level key: cloudWatchConfig. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchConfig": { + "logGroupName": "string", // result log group; omit for + // SOURCE_LOG_GROUP, and it cannot sit + // under /aws/bedrock-agentcore/evaluations/ + "logStreamName": "string", // result log stream + "metricsNamespace": "string", // defaults to + // Bedrock-AgentCore/Evaluations; cannot + // begin with "AWS/" + "resultDestination": "DEDICATED_LOG_GROUP" | "SOURCE_LOG_GROUP" + // DEDICATED_LOG_GROUP (default) writes to + // a dedicated result group; + // SOURCE_LOG_GROUP writes back to the log + // group the traces were read from + } + } + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_OutputConfig.html + +Example: + --output-config '{"cloudWatchConfig":{"logGroupName":"/company/agent-evaluations","metricsNamespace":"Company/AgentEvaluations","resultDestination":"DEDICATED_LOG_GROUP"}}'`; + +export class BatchOutputConfig { + static readonly flags = [ + flag( + "output-config", + "where results and metrics are written (JSON OutputConfig)", + z.string().optional(), + { group: "Result output:", help: outputConfigHelp }, + ), + ] as const; + + static async resolve(value: string | undefined, io: AppIO): Promise { + const resolver = new SourceResolver({ stdin: io.stdin }); + return parseJsonFlag( + "output-config", + await resolver.resolveText("output-config", value), + ); + } +} diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index ff6c51ec7..23ee6d69c 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -6,6 +6,7 @@ import type { AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; +import { BatchOutputConfig } from "../outputConfig"; const RUNTIME_INVOCATION = "Runtime invocation:"; const DATASET = "Dataset:"; @@ -23,7 +24,7 @@ Example: // Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror // `runtime invoke`. -export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => +export const createSimulateBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ name: "simulate", description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions", @@ -31,7 +32,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), { group: RUNTIME_INVOCATION, }), - flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { + flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { group: RUNTIME_INVOCATION, }), flag( @@ -77,6 +78,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { group: "Evaluation:", }), + ...BatchOutputConfig.flags, ], handle: async (ctx, flags) => { if (!flags["runtime-id"]) @@ -96,6 +98,8 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). // TODO(#1986): swap for the shared SIGINT/abort helper once it merges. + const outputConfig = await BatchOutputConfig.resolve(flags["output-config"], io); + const controller = new AbortController(); const interrupt = () => controller.abort(); process.once("SIGINT", interrupt); @@ -105,7 +109,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => const r = await core.eval.invokeDataset( { runtimeId: flags["runtime-id"], - qualifier: flags["qualifier"], + qualifier: flags["endpoint"], payloadTemplate: flags["payload-template"], headers: parseRuntimeInvokeHeaders(flags["header"]), bearerToken: flags["bearer-token"], @@ -134,7 +138,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => source: { origin: "agent", agent: flags["runtime-id"], - endpoint: flags["qualifier"], + endpoint: flags["endpoint"], sessionIds: r.sessions.map((s) => s.sessionId), }, groundTruth: r.sessions.map((s) => ({ @@ -143,6 +147,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => ...(s.groundTruth && { groundTruth: { inline: s.groundTruth } }), })), kmsKeyArn: flags["kms-key-arn"], + outputConfig, }, opts, ); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 8d84c3656..7c8dd4c29 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -51,6 +51,7 @@ import type { InlineGroundTruth, EvaluationReferenceInput, EvaluationResultContent, + OutputConfig, DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; @@ -289,6 +290,7 @@ export type StartBatchEvaluationInput = { // Already-parsed --ground-truth (SessionMetadataShape[]) → evaluationMetadata. groundTruth?: SessionMetadataShape[]; kmsKeyArn?: string; + outputConfig?: OutputConfig; }; // Batch insights use the same service job API as batch evaluations, but remain From 50a675355e715fbc6deb0b0689df58f246d08812 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 10 Sep 2026 23:48:42 +0000 Subject: [PATCH 2/2] feat(online-eval): add --output-config and --tags, --description on update --output-config lets a customer say where results and metrics are written. Same passthrough contract as batch evaluation's flag of the same name, but a separate module and a separate generated type: the online-evaluation OutputConfig is a plain object with no logStreamName, while the batch one is a tagged union that has one. --tags on create reuses TagsSchema and parseJsonFlagWithSchema, matching `project add memory`, so a non-string tag value is rejected here rather than by the API. --description on update was simply missing. The execution role is what makes the destination usable. Its policy granted result writes only to /aws/bedrock-agentcore/evaluations/*, so a CLI-managed role plus a custom log group would have produced a config whose results could not be written anywhere. executionPolicy now also grants the customer-named group, or the sampled source groups for SOURCE_LOG_GROUP, and an output change joins a data-source change as a trigger for re-scoping. A single ARN stays a bare string rather than a one-element array. IAM treats those identically, but the document's exact text is hashed to name the inline policy, so wrapping it would rename the policy attached to every config that has no custom destination. A destination already inside the reserved /aws/bedrock-agentcore/evaluations/ namespace adds nothing, keeping the document stable for a config whose stored outputConfig is the service-managed default the API echoes back. RoleScopeWarning carries which half of the scope moved, because the permissions the caller has to add differ: querying traces needs logs:StartQuery and logs:GetQueryResults on the sampled groups, while writing results needs logs:PutLogEvents and, for a group that does not exist yet, logs:CreateLogGroup. --- src/core/eval.tsx | 53 +++++- src/core/onlineEvalExecutionRole.test.ts | 65 +++++++ src/core/onlineEvalExecutionRole.tsx | 41 +++- .../eval/online-eval/create/index.tsx | 30 ++- .../online-eval/online-eval.flags.test.tsx | 178 ++++++++++++++++++ .../eval/online-eval/outputConfig.tsx | 58 ++++++ .../eval/online-eval/update/index.tsx | 34 +++- src/handlers/eval/types.tsx | 15 +- src/testing/TestCoreClient.tsx | 17 +- 9 files changed, 467 insertions(+), 24 deletions(-) create mode 100644 src/handlers/eval/online-eval/online-eval.flags.test.tsx create mode 100644 src/handlers/eval/online-eval/outputConfig.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 61bd76802..28b417c3c 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -123,6 +123,8 @@ import type { DatasetUpdateProgressEvent, DatasetUpdateResult, RoleScopeWarning, + RoleScopeKind, + OnlineEvalOutputConfig, CoreEvalClient, CreateConfigurationBundleInput, CreateConfigBasedABTestInput, @@ -989,6 +991,9 @@ export class EvalClient implements CoreEvalClient { options.region, logGroupNamesOf(dataSourceConfig), await evaluatorKmsKeys(input.evaluatorIds ?? [], control), + // Read only to widen the write scope to the chosen destination; the + // request object below still gets the caller's object untouched. + { outputConfig: input.outputConfig }, ) ).roleArn; @@ -998,8 +1003,10 @@ export class EvalClient implements CoreEvalClient { rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), dataSourceConfig, evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + outputConfig: input.outputConfig, evaluationExecutionRoleArn, enableOnCreate: input.enableOnCreate ?? true, + tags: input.tags, }); // A role provisioned moments ago may not be assumable yet (IAM is eventually @@ -1237,6 +1244,11 @@ export class EvalClient implements CoreEvalClient { ? dataSourceConfig : undefined; + const outputMoved = update.outputConfig !== undefined; + const effectiveOutputConfig = update.outputConfig ?? current.outputConfig; + const scopeKind: RoleScopeKind = + movedTo !== undefined && outputMoved ? "input-and-output" : outputMoved ? "output" : "input"; + const configName = current.onlineEvaluationConfigName; const roleArn = update.evaluationExecutionRoleArn ?? current.evaluationExecutionRoleArn; const managedRoleName = @@ -1246,26 +1258,33 @@ export class EvalClient implements CoreEvalClient { isManagedOnlineEvalRole(roleArn, configName) ? configName : undefined; - const refreshManagedRole = movedTo !== undefined && managedRoleName !== undefined; - - if (movedTo !== undefined && managedRoleName === undefined && roleArn) { + const scopeChanged = movedTo !== undefined || outputMoved; + const refreshManagedRole = scopeChanged && managedRoleName !== undefined; + const affectedLogGroups = [ + ...(movedTo !== undefined ? logGroupNamesOf(movedTo) : []), + ...(outputMoved ? destinationLogGroupNames(update.outputConfig, dataSourceConfig) : []), + ]; + + if (scopeChanged && managedRoleName === undefined && roleArn) { roleScopeWarning = { reason: "custom-role", roleArn, - logGroupNames: logGroupNamesOf(movedTo), + scope: scopeKind, + logGroupNames: affectedLogGroups, }; - } else if (movedTo !== undefined && !refreshManagedRole && roleArn) { + } else if (scopeChanged && !refreshManagedRole && roleArn) { // managed role, but the caller declined the refresh roleScopeWarning = { reason: "update-declined", roleArn, - logGroupNames: logGroupNamesOf(movedTo), + scope: scopeKind, + logGroupNames: affectedLogGroups, }; } if (refreshManagedRole && update.updateRole !== false) { const iam = this.clients.iam({ region: options.region }); - const newLogGroups = logGroupNamesOf(movedTo); + const newLogGroups = dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : []; const oldLogGroups = current.dataSourceConfig ? logGroupNamesOf(current.dataSourceConfig) : []; @@ -1290,7 +1309,7 @@ export class EvalClient implements CoreEvalClient { options.region, newLogGroups, kmsKeys, - resourceNameFromArn(roleArn!), + { roleName: resourceNameFromArn(roleArn!), outputConfig: effectiveOutputConfig }, ); const oldPolicyName = scopePolicyName( executionPolicy( @@ -1298,15 +1317,18 @@ export class EvalClient implements CoreEvalClient { accountIdFromRoleArn(managedRoleArn), oldLogGroups, kmsKeys, + current.outputConfig, ), ); const response = await control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, + description: update.description, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), dataSourceConfig, evaluators, + outputConfig: update.outputConfig, }), ); @@ -1323,6 +1345,7 @@ export class EvalClient implements CoreEvalClient { roleScopeWarning = { reason: "stale-scope", roleArn: roleArn!, + scope: scopeKind, logGroupNames: oldLogGroups, }; } @@ -1333,9 +1356,11 @@ export class EvalClient implements CoreEvalClient { const response = await control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, + description: update.description, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), dataSourceConfig, evaluators, + outputConfig: update.outputConfig, evaluationExecutionRoleArn: update.evaluationExecutionRoleArn, }), ); @@ -2201,6 +2226,18 @@ function logGroupNamesOf(dataSourceConfig: DataSourceConfig): string[] { : []; } +function destinationLogGroupNames( + outputConfig: OnlineEvalOutputConfig | undefined, + dataSourceConfig: DataSourceConfig | undefined, +): string[] { + const cloudWatch = outputConfig?.cloudWatchConfig; + if (!cloudWatch) return []; + if (cloudWatch.resultDestination === "SOURCE_LOG_GROUP") { + return dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : []; + } + return cloudWatch.logGroupName ? [cloudWatch.logGroupName] : []; +} + // runtimeIdFromLogGroup recovers the runtime id embedded in a log group path // produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for a // new --endpoint without the caller passing --agent again. Returns undefined for diff --git a/src/core/onlineEvalExecutionRole.test.ts b/src/core/onlineEvalExecutionRole.test.ts index 7f1ec01ea..e4a719410 100644 --- a/src/core/onlineEvalExecutionRole.test.ts +++ b/src/core/onlineEvalExecutionRole.test.ts @@ -110,3 +110,68 @@ test("gives identical policies the same name", () => { scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])), ); }); + +function writeStatement(policy: string) { + return statements(policy).find((s) => s.Sid === "WriteEvaluationResults"); +} + +const SERVICE_RESULTS = `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/evaluations/*`; + +test("a config with no output destination keeps the service namespace as a bare string", () => { + const write = writeStatement(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])); + + expect(write?.Resource).toBe(SERVICE_RESULTS); + expect(Array.isArray(write?.Resource)).toBe(false); +}); + +test("a customer-named dedicated group is granted alongside the service namespace", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/company/agent-evaluations*`, + ]); + expect(write?.Action).toContain("logs:CreateLogGroup"); +}); + +test("SOURCE_LOG_GROUP grants writes to the groups the traces are read from", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`, + ]); +}); + +test("a destination already inside the service namespace adds nothing", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/aws/bedrock-agentcore/evaluations/online-evaluations/results/default", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toBe(SERVICE_RESULTS); +}); + +test("changing the destination changes the policy name, so a re-scope is a new grant", () => { + const before = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []); + const after = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { logGroupName: "/company/agent-evaluations" }, + }); + + expect(scopePolicyName(after)).not.toBe(scopePolicyName(before)); +}); diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx index e02a6f40a..4c1896b6e 100644 --- a/src/core/onlineEvalExecutionRole.tsx +++ b/src/core/onlineEvalExecutionRole.tsx @@ -15,7 +15,6 @@ import { parseArn, resourceNameFromArn } from "./arn"; // evaluation results back to CloudWatch. When the caller doesn't bring one, // OnlineEvalClient provisions a per-config default here, scoped to the log // group(s) being sampled. Idempotent: an existing role is reused. -// // Each scope is stored as its own inline policy, named after a fingerprint of the // scope, so granting a new scope never overwrites the policy backing the current // one. IAM unions Allows across a role's inline policies, which lets an update @@ -83,10 +82,35 @@ function runtimeLogGroupPrefix(logGroupName: string): string { return match?.[1] ?? logGroupName; } +const SERVICE_RESULT_PREFIX = "/aws/bedrock-agentcore/evaluations/"; + +function resultWriteArns( + logs: string, + sampledArns: string[], + outputConfig: OnlineEvalResultDestination | undefined, +): string | string[] { + const arns = [`${logs}:${SERVICE_RESULT_PREFIX}*`]; + const cloudWatch = outputConfig?.cloudWatchConfig; + + if (cloudWatch?.resultDestination === "SOURCE_LOG_GROUP") { + arns.push(...sampledArns); + } else if ( + cloudWatch?.logGroupName && + !cloudWatch.logGroupName.startsWith(SERVICE_RESULT_PREFIX) + ) { + arns.push(`${logs}:${cloudWatch.logGroupName}*`); + } + + return arns.length === 1 ? arns[0]! : arns; +} + +export type OnlineEvalResultDestination = { + cloudWatchConfig?: { logGroupName?: string; resultDestination?: string } | undefined; +}; + // executionPolicy grants the permissions CreateOnlineEvaluationConfig validates // at creation time. Exported for assertion: the policy body is not observable // through the recorded IAM fixtures, whose responses are empty. -// // at creation time: Logs Insights query access over the sampled log groups plus // the `aws/spans` group that carries the actual trace spans, Bedrock model // invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based @@ -98,6 +122,7 @@ export function executionPolicy( accountId: string, logGroupNames: string[], kmsKeyArns: string[], + outputConfig?: OnlineEvalResultDestination, ): string { const logs = `arn:aws:logs:${region}:${accountId}:log-group`; const spansArn = `${logs}:aws/spans`; @@ -134,6 +159,8 @@ export function executionPolicy( Resource: [`${spansArn}*`, ...sampledArns], }, { + // logs:CreateLogGroup is needed because the service creates a + // customer-named result group that does not exist yet. Sid: "WriteEvaluationResults", Effect: "Allow", Action: [ @@ -142,7 +169,7 @@ export function executionPolicy( "logs:DescribeLogStreams", "logs:PutLogEvents", ], - Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`, + Resource: resultWriteArns(logs, sampledArns, outputConfig), }, { Sid: "IndexSpans", @@ -201,6 +228,11 @@ export function scopePolicyName(policyDocument: string): string { return `${POLICY_PREFIX}-${fingerprint(policyDocument)}`; } +export type GrantScopeOptions = { + roleName?: string; + outputConfig?: OnlineEvalResultDestination; +}; + // grantOnlineEvalScope creates the execution role for `configName` if it does not // exist and attaches the inline policy for this scope, returning the role ARN and // the policy name written. The caller revokes the superseded scope once whatever @@ -211,7 +243,7 @@ export async function grantOnlineEvalScope( region: string, logGroupNames: string[], kmsKeyArns: string[] = [], - roleName = onlineEvalExecutionRoleName(configName), + { roleName = onlineEvalExecutionRoleName(configName), outputConfig }: GrantScopeOptions = {}, ): Promise<{ roleArn: string; policyName: string }> { let roleArn: string; try { @@ -234,6 +266,7 @@ export async function grantOnlineEvalScope( accountIdFromRoleArn(roleArn), logGroupNames, kmsKeyArns, + outputConfig, ); const policyName = scopePolicyName(policyDocument); await iam.send( diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index 273b95c7b..c28b7bb49 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -5,9 +5,24 @@ import { InputValidationError } from "../../../../errors"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; -import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { + assertMutuallyExclusiveFlags, + coreOptsFromCtx, + parseJsonFlag, + parseJsonFlagWithSchema, +} from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; +import { TagsSchema } from "../../../../projectSchemas/tags"; + +const tagsHelp = `(JSON: map of string to string) +Tags applied to the online evaluation configuration. + +Accepts inline JSON, file://, or - to read stdin. + +Example: + --tags '{"team":"ml-platform","env":"prod"}'`; const CONFIGURATION = "Configuration:"; const SESSION_SOURCE = "Session source (choose exactly one):"; @@ -35,6 +50,10 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => z.enum(["true", "false"]).optional(), { group: CONFIGURATION }, ), + flag("tags", "resource tags (JSON object of key/value strings)", z.string().optional(), { + group: CONFIGURATION, + help: tagsHelp, + }), flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { group: SESSION_SOURCE, }), @@ -69,6 +88,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", @@ -98,9 +118,17 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => } const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], io); + const tags = parseJsonFlagWithSchema( + "tags", + await source.resolveText("tags", flags["tags"]), + TagsSchema, + ); const common = { name: flags["name"], description: flags["description"], + tags, + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( diff --git a/src/handlers/eval/online-eval/online-eval.flags.test.tsx b/src/handlers/eval/online-eval/online-eval.flags.test.tsx new file mode 100644 index 000000000..e198386ce --- /dev/null +++ b/src/handlers/eval/online-eval/online-eval.flags.test.tsx @@ -0,0 +1,178 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; +import type { CreateOnlineEvalInput, UpdateOnlineEvalInput } from "../types"; + +async function run( + args: string[], + configure?: (core: TestCoreClient) => void, + ioOptions?: { stdin?: string }, +) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(ioOptions); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +function createInput(core: TestCoreClient): CreateOnlineEvalInput { + const call = core.eval.calls.find((c) => c.method === "createOnlineEvaluationConfig"); + expect(call).toBeDefined(); + return call!.args[0] as CreateOnlineEvalInput; +} + +function updateInput(core: TestCoreClient): UpdateOnlineEvalInput { + const call = core.eval.calls.find((c) => c.method === "updateOnlineEvaluationConfig"); + expect(call).toBeDefined(); + return call!.args[1] as UpdateOnlineEvalInput; +} + +const OUTPUT_CONFIG = { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + metricsNamespace: "Company/AgentEvaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, +} as const; + +describe("eval online-eval create", () => { + const BASE = [ + "eval", + "online-eval", + "create", + "--name", + "quality", + "--agent", + "r-1", + "--evaluators", + "Builtin.Helpfulness", + "--sampling-rate", + "10", + ]; + + test("--output-config reaches Core unchanged from inline JSON", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(OUTPUT_CONFIG)]); + expect(createInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("--output-config reaches Core unchanged from stdin", async () => { + const { core } = await run([...BASE, "--output-config", "-"], undefined, { + stdin: JSON.stringify(OUTPUT_CONFIG), + }); + expect(createInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("--tags reaches Core as a parsed map", async () => { + const { core } = await run([...BASE, "--tags", '{"team":"ml-platform","env":"dev"}']); + expect(createInput(core).tags).toEqual({ team: "ml-platform", env: "dev" }); + }); + + test("both are left undefined when omitted, so the service keeps its defaults", async () => { + const { core } = await run(BASE); + expect(createInput(core).outputConfig).toBeUndefined(); + expect(createInput(core).tags).toBeUndefined(); + }); + + test.each([ + ["--output-config", "{not json", /Invalid JSON for option '--output-config'/], + ["--tags", "{not json", /Invalid JSON for option '--tags'/], + ])("malformed %s fails before Core provisions anything", async (flag, value, expected) => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route(["node", "agentcore", ...BASE, flag, value, "--region", "us-west-2"]), + ).rejects.toThrow(expected); + expect(core.eval.calls).toEqual([]); + }); + + test("--tags rejects a non-string value rather than passing it to the API", async () => { + await expect(run([...BASE, "--tags", '{"team":42}'])).rejects.toThrow( + /Invalid value for option '--tags'/, + ); + }); +}); + +describe("eval online-eval update", () => { + const BASE = ["eval", "online-eval", "update", "--id", "online-eval-123"]; + + test("--description reaches Core", async () => { + const { core } = await run([...BASE, "--description", "checks tone on prod traffic"]); + expect(updateInput(core).description).toBe("checks tone on prod traffic"); + }); + + test("--output-config reaches Core unchanged", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(OUTPUT_CONFIG)]); + expect(updateInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("an omitted --output-config is not sent, so the destination is preserved", async () => { + const { core } = await run([...BASE, "--sampling-rate", "20"]); + expect(updateInput(core).outputConfig).toBeUndefined(); + }); + + test("malformed --output-config fails before the initial Get", async () => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route([ + "node", + "agentcore", + ...BASE, + "--output-config", + "{not json", + "--region", + "us-west-2", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + expect(core.eval.calls).toEqual([]); + }); + + test.each([ + ["input", "data source moved", "logs:StartQuery and logs:GetQueryResults"], + ["output", "output destination moved", "logs:PutLogEvents"], + ["input-and-output", "data source and output destination moved", "logs:PutLogEvents"], + ] as const)( + "a %s scope warning names what moved and what to grant", + async (scope, movedText, action) => { + const { stderr } = await run(BASE, (c) => + c.eval.setOnlineEvalRoleScopeWarning({ + reason: "custom-role", + roleArn: "arn:aws:iam::123456789012:role/MyRole", + scope, + logGroupNames: ["/company/agent-evaluations"], + }), + ); + expect(stderr).toContain(movedText); + expect(stderr).toContain(action); + expect(stderr).toContain("/company/agent-evaluations"); + }, + ); + + test("the scope warning is suppressed under --json", async () => { + const { stderr } = await run([...BASE, "--json"], (c) => + c.eval.setOnlineEvalRoleScopeWarning({ + reason: "custom-role", + roleArn: "arn:aws:iam::123456789012:role/MyRole", + scope: "output", + logGroupNames: ["/company/agent-evaluations"], + }), + ); + expect(stderr).toBe(""); + }); +}); diff --git a/src/handlers/eval/online-eval/outputConfig.tsx b/src/handlers/eval/online-eval/outputConfig.tsx new file mode 100644 index 000000000..b958592ae --- /dev/null +++ b/src/handlers/eval/online-eval/outputConfig.tsx @@ -0,0 +1,58 @@ +import z from "zod"; +import { SourceResolver, type AppIO } from "../../../io"; +import { flag } from "../../../router"; +import { parseJsonFlag } from "../../utils"; +import type { OnlineEvalOutputConfig } from "../types"; + +const outputConfigHelp = `(JSON object) +Where evaluation results and metrics are written. Omit it and results go to the +service-managed default location. Only top-level key: cloudWatchConfig. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchConfig": { + "logGroupName": "string", // result log group; omit for + // SOURCE_LOG_GROUP, and it cannot sit + // under /aws/bedrock-agentcore/evaluations/ + "metricsNamespace": "string", // CloudWatch metrics namespace + "resultDestination": "DEDICATED_LOG_GROUP" | "SOURCE_LOG_GROUP" + // DEDICATED_LOG_GROUP writes to a + // dedicated result group, creating it if + // needed; SOURCE_LOG_GROUP writes back to + // the groups the traces were sampled from + } + } + +A role you supply with --role-arn is never edited by the CLI, so it must already +grant logs:PutLogEvents on the destination — plus logs:CreateLogGroup when the +group does not exist yet. + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore-control/latest/APIReference/API_OutputConfig.html + +Example: + --output-config '{"cloudWatchConfig":{"logGroupName":"/company/agent-evaluations","metricsNamespace":"Company/AgentEvaluations","resultDestination":"DEDICATED_LOG_GROUP"}}'`; + +export class OnlineEvalOutputConfigFlag { + static readonly flags = [ + flag( + "output-config", + "where results and metrics are written (JSON OutputConfig)", + z.string().optional(), + { group: "Result output:", help: outputConfigHelp }, + ), + ] as const; + + static async resolve( + value: string | undefined, + io: AppIO, + ): Promise { + const resolver = new SourceResolver({ stdin: io.stdin }); + return parseJsonFlag( + "output-config", + await resolver.resolveText("output-config", value), + ); + } +} diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 6bdaa6807..57d932e30 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -6,15 +6,31 @@ import { JsonKey } from "../../../keys"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; +import type { RoleScopeKind } from "../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; const SESSION_SOURCE = "Session source:"; const SOURCE_FILTERS = "Source filters:"; const EVALUATION = "Evaluation:"; const EXECUTION = "Execution:"; +const MOVED: Record = { + input: "data source", + output: "output destination", + "input-and-output": "data source and output destination", +}; + +const QUERY_ACTIONS = "logs:StartQuery and logs:GetQueryResults"; +const WRITE_ACTIONS = "logs:CreateLogGroup, logs:CreateLogStream and logs:PutLogEvents"; +const NEEDED: Record = { + input: QUERY_ACTIONS, + output: WRITE_ACTIONS, + "input-and-output": `${QUERY_ACTIONS}, plus ${WRITE_ACTIONS}`, +}; + export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "update", @@ -23,6 +39,12 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => flag("id", "the ID of the online evaluation config to update", z.string().optional(), { group: "Target:", }), + flag( + "description", + "replace the description of the config's monitoring purpose", + z.string().optional(), + { group: "Configuration:" }, + ), flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { group: SESSION_SOURCE, }), @@ -66,6 +88,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "replace the IAM role the online evaluation assumes", @@ -74,7 +97,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => ), flag( "update-role", - "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", + "whether to re-scope an auto-provisioned execution role when the data source or output destination changes (default true)", z.enum(["true", "false"]).optional(), { group: EXECUTION }, ), @@ -97,9 +120,12 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => } const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], io); const { response, roleScopeWarning } = await core.eval.updateOnlineEvaluationConfig( flags["id"], { + description: flags["description"], + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( @@ -123,7 +149,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => // Suppressed under --json, matching runtime/invoke's advisory summary: a // scripted caller gets a machine-readable stdout and nothing else. if (roleScopeWarning && !ctx.require(JsonKey)) { - const { reason, roleArn, logGroupNames } = roleScopeWarning; + const { reason, roleArn, scope, logGroupNames } = roleScopeWarning; if (reason === "stale-scope") { // The update succeeded and the role grants the new data source; the // policy for the superseded one just could not be detached. @@ -138,9 +164,9 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => ? "it is not managed by the CLI" : "re-scoping was declined via --update-role false"; io.stderr.write( - `warning: the data source moved but the execution role was not re-scoped because ${detail}.\n` + + `warning: the ${MOVED[scope]} moved but the execution role was not re-scoped because ${detail}.\n` + ` role: ${roleArn}\n` + - ` ensure it grants logs:StartQuery and logs:GetQueryResults on: ${logGroupNames.join(", ")}\n`, + ` ensure it grants ${NEEDED[scope]} on: ${logGroupNames.join(", ")}\n`, ); } } diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 7c8dd4c29..79e5e4afb 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -28,6 +28,7 @@ import type { UpdateConfigurationBundleResponse, UpdateEvaluatorResponse, UpdateOnlineEvaluationConfigResponse, + OutputConfig as OnlineEvalOutputConfig, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { CreateABTestResponse, @@ -157,6 +158,8 @@ export type CreateOnlineEvalInput = { evaluatorIds?: string[]; evaluationExecutionRoleArn?: string; enableOnCreate?: boolean; + tags?: Record; + outputConfig?: OnlineEvalOutputConfig; } & ( | { agent: string; endpoint?: string; dataSourceConfig?: undefined } | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } @@ -204,6 +207,7 @@ export type DeleteOnlineInsightResponse = DeleteOnlineEvaluationConfigResponse; // `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to // the agent's default log group. export type UpdateOnlineEvalInput = { + description?: string; samplingRate?: number; sessionTimeoutMinutes?: number; filters?: Rule["filters"]; @@ -219,21 +223,22 @@ export type UpdateOnlineEvalInput = { // Replaces the execution role. The CLI never edits the permissions of a role the // caller names here — it is theirs to manage. evaluationExecutionRoleArn?: string; - // Whether to re-scope a CLI-provisioned role when the data source moves - // (default true). Only meaningful for a managed role: the old policy grants - // query access to the previous log groups only. + outputConfig?: OnlineEvalOutputConfig; updateRole?: boolean; }; // RoleScopeWarning reports that an execution role was left scoped to log groups -// the config no longer samples, so the caller can surface it. Returned rather -// than logged from Core so the handler owns how it is presented. export type RoleScopeWarning = { reason: "custom-role" | "update-declined" | "stale-scope"; roleArn: string; + scope: RoleScopeKind; logGroupNames: string[]; }; +export type RoleScopeKind = "input" | "output" | "input-and-output"; + +export type { OnlineEvalOutputConfig }; + export type BundleRef = { configBundle: string; bundleVersion: string }; export type CreateConfigBasedABTestInput = { diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index cf3bdd18e..df13f5e8b 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -185,6 +185,7 @@ import type { StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, + RoleScopeWarning, } from "../handlers/eval/types"; import { isTerminalStatus } from "../core/batchEvaluationResults"; import { abortable } from "../core/abortable"; @@ -1532,6 +1533,7 @@ export class TestEvalClient implements CoreEvalClient { DEFAULT_CREATE_ONLINE_EVAL_RESPONSE; private onlineEvalUpdateResponse: UpdateOnlineEvaluationConfigResponse = DEFAULT_UPDATE_ONLINE_EVAL_RESPONSE; + private onlineEvalRoleScopeWarning: RoleScopeWarning | undefined; private onlineEvalGetResponse: GetOnlineEvaluationConfigResponse = DEFAULT_GET_ONLINE_EVAL_RESPONSE; private onlineEvalDeleteResponse: DeleteOnlineEvaluationConfigResponse = @@ -1668,6 +1670,11 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setOnlineEvalRoleScopeWarning(warning: RoleScopeWarning): this { + this.onlineEvalRoleScopeWarning = warning; + return this; + } + // setOnlineEvalGetResponse sets what getOnlineEvaluationConfig resolves to // (when not erroring). setOnlineEvalGetResponse(response: GetOnlineEvaluationConfigResponse): this { @@ -2122,10 +2129,16 @@ export class TestEvalClient implements CoreEvalClient { id: string, update: UpdateOnlineEvalInput, options: CoreOptions, - ): Promise<{ response: UpdateOnlineEvaluationConfigResponse }> { + ): Promise<{ + response: UpdateOnlineEvaluationConfigResponse; + roleScopeWarning?: RoleScopeWarning; + }> { this.calls.push({ method: "updateOnlineEvaluationConfig", args: [id, update, options] }); if (this.error) throw this.error; - return { response: this.onlineEvalUpdateResponse }; + return { + response: this.onlineEvalUpdateResponse, + roleScopeWarning: this.onlineEvalRoleScopeWarning, + }; } async getOnlineEvaluationConfig(