From 7538097142f44140a437c0408d693934bf398f1f Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 9 Sep 2026 22:20:11 +0000 Subject: [PATCH 1/5] feat(router): add help groups and command examples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two presentation mechanisms for commands whose option list has grown past the point of skimming. No command uses either yet — the follow-up PRs that add flags to batch-evaluation and online-evaluation consume them. Help groups: - Flag.group renders through Commander's Option.helpGroup(), so a command can carry semantic headings instead of one flat "Options:". Ungrouped flags keep Commander's default section, so nothing changes for commands that opt out. - A command that groups its flags gets its generated -h/--help moved to "Other options:". Left alone it sits by itself under "Options:", reading as though it were the command's only ungrouped flag. Examples: - Handler.examples takes { description, command } entries and the renderer owns the layout. `command` is authored as the shell command with no indentation: a string renders on one line, an array joins with backslash continuations. Splitting the array is how the author picks the break points, which keeps a terminal-width heuristic out of it and keeps the printed command pasteable. - commandExamples() mirrors commandParameterDetails(): Commander omits added help text from helpInformation(), so a TUI rendering has to ask for it or silently drop it. --- src/router/flags.tsx | 29 ++++++++- src/router/handler.tsx | 39 ++++++++++- src/router/index.tsx | 2 + src/router/router.test.ts | 133 ++++++++++++++++++++++++++++++++++++++ src/router/router.tsx | 40 +++++++++++- 5 files changed, 237 insertions(+), 6 deletions(-) diff --git a/src/router/flags.tsx b/src/router/flags.tsx index 6314cf223..72f8945e6 100644 --- a/src/router/flags.tsx +++ b/src/router/flags.tsx @@ -1,7 +1,7 @@ import { Option } from "commander"; import { InputValidationError } from "../errors"; import type { Context } from "./context"; -import type { Flag, GlobalFlag } from "./handler"; +import type { Example, Flag, GlobalFlag } from "./handler"; import { coerce, formatZodError, inspect } from "./schema"; // toOption builds a Commander Option from a flag's schema. A boolean that defaults @@ -32,6 +32,9 @@ export function toOption(flag: Flag): Option { if (info.required && !info.boolean) { option.makeOptionMandatory(true); } + if (flag.group) { + option.helpGroup(flag.group); + } return option; } @@ -53,6 +56,30 @@ export function formatParameterDetails(flags: Flag[]): string | undefined { return `\nParameter details:\n\n${sections.join("\n\n")}\n`; } +// Indentation of a rendered example: the description sits one level in, the +// command another, and a continued command line one level deeper again so the +// backslash-joined flags read as belonging to the line above. +const EXAMPLE_DESCRIPTION_INDENT = " "; +const EXAMPLE_COMMAND_INDENT = " "; +const EXAMPLE_CONTINUATION_INDENT = " "; + +// formatExamples renders a command's worked invocations into the block appended +// after the option list (and after Parameter details, when present). Authors +// supply the shell command only; the layout lives here so every command's +// examples line up identically no matter who wrote them. +export function formatExamples(examples: readonly Example[]): string | undefined { + if (examples.length === 0) return undefined; + + const blocks = examples.map(({ description, command }) => { + const body = Array.isArray(command) + ? command.join(` \\\n${EXAMPLE_CONTINUATION_INDENT}`) + : command; + return `${EXAMPLE_DESCRIPTION_INDENT}${description}:\n\n${EXAMPLE_COMMAND_INDENT}${body}`; + }); + + return `\nExamples:\n\n${blocks.join("\n\n")}\n`; +} + // attributeName mirrors how Commander camelCases an option name into the key it // stores on the parsed options object (e.g. "harness-id" -> "harnessId"). function attributeName(name: string): string { diff --git a/src/router/handler.tsx b/src/router/handler.tsx index 146c6dff2..d634a7f0e 100644 --- a/src/router/handler.tsx +++ b/src/router/handler.tsx @@ -14,10 +14,26 @@ export interface Flag { // Its first line is the type annotation shown next to the flag name; the // remaining lines are the body — prose, JSON syntax, examples. help?: string; + // group is the `--help` heading this flag is listed under. Commands whose + // option list is long enough to skim past benefit from semantic headings + // ("Session source:", "Result output:") over one flat "Options:" block. + // Ungrouped flags stay in Commander's default section. + group?: string; // sensitive flags are redacted from debug logs by the withLogging middleware. sensitive?: boolean; } +// Example is one worked invocation shown in a command's `--help`. `command` is +// authored as the shell command itself, with no indentation or line +// continuations: a string renders on one line, and an array renders joined by +// ` \` + newline. Splitting the array is how the author chooses where the breaks +// fall — usually one flag per element — so the renderer never has to guess at a +// terminal width, and the printed command still pastes into a shell verbatim. +export interface Example { + description: string; + command: string | string[]; +} + // GlobalFlag is a group-level flag that is *also* a typed ContextKey: declared on // a Router, its validated value is injected into the context under itself, so any // descendant handler can retrieve it type-safely via `ctx.value(theGlobalFlag)`. @@ -34,9 +50,16 @@ export function flag( name: N, description: string, schema: z.ZodType, - options?: { help?: string; sensitive?: boolean }, + options?: { help?: string; group?: string; sensitive?: boolean }, ): Flag { - return { name, description, schema, help: options?.help, sensitive: options?.sensitive }; + return { + name, + description, + schema, + help: options?.help, + group: options?.group, + sensitive: options?.sensitive, + }; } // globalFlag constructs a GlobalFlag. The returned value doubles as the typed @@ -85,6 +108,11 @@ export interface Handler { arguments(): Argument[]; // Middleware must preserve this metadata when wrapping a handler. doesSupportTui(): boolean; + // examples are the worked invocations appended to `--help` after the option + // list. Optional because only the handler that authors examples needs to + // answer it: compile() reads it off the authored node, so the middleware + // wrappers (which only forward `handle`) never have to carry it. + examples?(): readonly Example[] | undefined; // At runtime `handle` receives the validated, coerced flags object. The precise // shape is supplied to authors via createHandler's generic; the interface keeps // it erased so middleware can forward it uniformly. @@ -100,6 +128,7 @@ type CreateHandlerInput< description: string; flags?: F; arguments?: A; + examples?: readonly Example[]; handle?: HandleFn; children?: Handler[]; }; @@ -111,6 +140,7 @@ class BaseHandler implements Handler { _description: string; _flags: Flag[]; _arguments: Argument[]; + _examples?: readonly Example[]; _handle: HandleFn; _children: Handler[]; @@ -121,6 +151,7 @@ class BaseHandler implements Handler { this._description = input.description; this._flags = (input.flags ?? []) as Flag[]; this._arguments = (input.arguments ?? []) as Argument[]; + this._examples = input.examples; this._handle = (input.handle ?? noOpHandler) as HandleFn; this._children = input.children ?? []; } @@ -145,6 +176,10 @@ class BaseHandler implements Handler { return true; } + examples(): readonly Example[] | undefined { + return this._examples; + } + async handle(ctx: Context, flags: any, args: any): Promise { await this._handle(ctx, flags, args); } diff --git a/src/router/index.tsx b/src/router/index.tsx index f1c6b78a4..ad31dca93 100644 --- a/src/router/index.tsx +++ b/src/router/index.tsx @@ -13,9 +13,11 @@ export { isDefaultHandlerProvider, isTuiCommandSupported, commandParameterDetails, + commandExamples, } from "./router"; export { type Handler, + type Example, type Flag, type GlobalFlag, type Argument, diff --git a/src/router/router.test.ts b/src/router/router.test.ts index ecdc19726..31d04f155 100644 --- a/src/router/router.test.ts +++ b/src/router/router.test.ts @@ -899,3 +899,136 @@ test("--version is an unknown option on a router without a version", async () => code: "commander.unknownOption", }); }); + +test("grouped flags render under their headings, ungrouped ones under Options", async () => { + const evaluate = createHandler({ + name: "evaluate", + description: "", + flags: [ + flag("agent", "the agent", z.string().optional(), { group: "Session source:" }), + flag("start-time", "window start", z.string().optional(), { group: "Source filters:" }), + flag("end-time", "window end", z.string().optional(), { group: "Source filters:" }), + flag("name", "the name", z.string().optional()), + ], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(evaluate); + + const out = await helpOutput(root, ["app", "evaluate", "--help"]); + + expect(out).toContain("Session source:"); + expect(out).toContain("Source filters:"); + // Headings appear in the order their first flag was declared, and each flag + // sits under its own heading rather than in one flat list. + expect(out.indexOf("Session source:")).toBeLessThan(out.indexOf("Source filters:")); + expect(out.indexOf("--agent")).toBeGreaterThan(out.indexOf("Session source:")); + expect(out.indexOf("--start-time")).toBeGreaterThan(out.indexOf("Source filters:")); + expect(out.indexOf("--end-time")).toBeGreaterThan(out.indexOf("Source filters:")); + // An ungrouped flag keeps Commander's default heading. + expect(out.indexOf("--name")).toBeGreaterThan(out.indexOf("Options:")); +}); + +test("a command that groups its flags moves the generated help into Other options", async () => { + const grouped = createHandler({ + name: "grouped", + description: "", + flags: [flag("agent", "the agent", z.string().optional(), { group: "Session source:" })], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(grouped); + + const out = await helpOutput(root, ["app", "grouped", "--help"]); + + expect(out).toContain("Other options:"); + expect(out.indexOf("-h, --help")).toBeGreaterThan(out.indexOf("Other options:")); +}); + +test("a command without grouped flags leaves the generated help in Options", async () => { + const plain = createHandler({ + name: "plain", + description: "", + flags: [flag("id", "the id", z.string().optional())], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(plain); + + const out = await helpOutput(root, ["app", "plain", "--help"]); + + expect(out).not.toContain("Other options:"); + expect(out).toContain("-h, --help"); +}); + +test("handler examples render once, after the parameter details", async () => { + const create = createHandler({ + name: "create", + description: "", + flags: [ + flag("model", "model config (JSON)", z.string().optional(), { + help: `(JSON object)\nThe model configuration.`, + }), + ], + examples: [ + { description: "Create with a Bedrock model", command: `app create --model '{"a":1}'` }, + ], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(create); + + const out = await helpOutput(root, ["app", "create", "--help"]); + + expect(out).toContain("Examples:"); + expect(out).toContain(" Create with a Bedrock model:"); + expect(out).toContain(` app create --model '{"a":1}'`); + expect(out.indexOf("Examples:")).toBeGreaterThan(out.indexOf("Parameter details:")); + expect(out.split("Examples:").length - 1).toBe(1); +}); + +test("a multi-line example is joined with backslash continuations", async () => { + const evaluate = createHandler({ + name: "evaluate", + description: "", + examples: [ + { + description: "Evaluate a Runtime with two evaluators", + command: [ + "app evaluate", + "--agent my-runtime", + "--evaluators Builtin.Helpfulness Builtin.Correctness", + ], + }, + { description: "List what is already running", command: "app list" }, + ], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(evaluate); + + const out = await helpOutput(root, ["app", "evaluate", "--help"]); + + // The author supplies the shell command only; indentation and the trailing + // backslashes are the renderer's, so every command's examples line up. + expect(out).toContain( + [ + " Evaluate a Runtime with two evaluators:", + "", + " app evaluate \\", + " --agent my-runtime \\", + " --evaluators Builtin.Helpfulness Builtin.Correctness", + ].join("\n"), + ); + // A single-string command stays on one line. + expect(out).toContain(" List what is already running:\n\n app list"); +}); + +test("commands without examples have no Examples section", async () => { + const root = new Router("app"); + root.handler(leaf("get", () => {})); + + const out = await helpOutput(root, ["app", "get", "--help"]); + + expect(out).not.toContain("Examples:"); +}); diff --git a/src/router/router.tsx b/src/router/router.tsx index 7fdbf1da5..26ee8d638 100644 --- a/src/router/router.tsx +++ b/src/router/router.tsx @@ -1,10 +1,16 @@ import type { Argument, Flag, GlobalFlag, Handler } from "./handler"; import { type Middleware, type MiddlewareProvider, isMiddlewareProvider } from "./middleware"; import { type Context, type ContextKey, ValueContext, contextKey } from "./context"; -import { applyGlobalFlags, formatParameterDetails, parseFlags, toOption } from "./flags"; +import { + applyGlobalFlags, + formatExamples, + formatParameterDetails, + parseFlags, + toOption, +} from "./flags"; import { parseArguments, toCommanderArgument } from "./args"; -import { Command, CommanderError } from "commander"; +import { Command, CommanderError, Option } from "commander"; import { InputValidationError } from "../errors"; import type { Logger } from "../logging"; import type { GlobalConfigAccessor } from "../globalConfig"; @@ -48,6 +54,16 @@ export function commandParameterDetails(command: Command): string | undefined { : undefined; } +// commandExamples is the worked-invocation block `--help` appends after the +// parameter details; undefined when the command authors none. Same reason as +// commandParameterDetails for existing: added help text is absent from +// helpInformation(), so a TUI rendering has to ask for it separately. +export function commandExamples(command: Command): string | undefined { + if (!(command instanceof RoutedCommand)) return undefined; + const examples = command.handler.examples?.(); + return examples ? formatExamples(examples) : undefined; +} + interface TuiChildSupportProvider { supportsTuiCommand(commandName: string): boolean; } @@ -63,6 +79,7 @@ function withEffectiveTuiSupport(handler: Handler, supported: boolean): Handler flags: () => handler.flags(), arguments: () => handler.arguments(), doesSupportTui: () => supported, + examples: () => handler.examples?.(), handle: (ctx, flags, args) => handler.handle(ctx, flags, args), children: () => handler.children(), }; @@ -187,12 +204,29 @@ export function compile( declareFlags(c, ownFlags); declareArguments(c, node.arguments()); + // A command that groups its own flags would otherwise leave Commander's + // generated `-h, --help` alone in the default "Options:" heading, reading as + // if it were the command's only ungrouped option. Give it its own heading so + // every listed flag sits under a deliberate one. + if (ownFlags.some((f) => f.group)) { + c.addHelpOption( + new Option("-h, --help", "display help for command").helpGroup("Other options:"), + ); + } + // Flags with long-form documentation get a "Parameter details" section after - // the option list in `--help` output. + // the option list in `--help` output. Examples follow it: Commander emits + // added help text in registration order, and the worked invocations read as + // the closing section. const parameterDetails = formatParameterDetails(ownFlags); if (parameterDetails) { c.addHelpText("after", parameterDetails); } + const examples = node.examples?.(); + const renderedExamples = examples ? formatExamples(examples) : undefined; + if (renderedExamples) { + c.addHelpText("after", renderedExamples); + } const own = isMiddlewareProvider(node) ? node.middlewares() : []; const nextStack = [...stack, ...own]; From 4912e18f956e82d469a7e48da0d10ba5194d4b8c Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 9 Sep 2026 23:16:48 +0000 Subject: [PATCH 2/5] feat(eval): group and document the eval command help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the router's group and example mechanisms to the five eval commands whose option lists had grown past skimming. Presentation only — no flag is added, removed, or renamed, and no request changes. One shared vocabulary (src/handlers/eval/helpGroups.tsx) rather than per-command headings: batch evaluation, batch insights, and online evaluation all name a session source, narrow it, and apply evaluators, so the same concept should read the same in all three. Naming the headings in one place also means a typo cannot silently split one heading into two. Notable placements: - `--name`/`--description`/`--kms-key-arn` sit under "Configuration:" rather than an "Evaluation options:" heading that would also have to hold encryption. - `--endpoint` joins "Session source:" on online-eval create instead of taking a heading of its own for one flag. - SessionSource declares its source arms before its filters, because Commander orders headings by the first flag declared in each. Its descriptions lose the "source:"/"filter:"/"time filter:" prefixes, which the headings now carry. Examples are worked invocations per command, covering each source arm and each authentication and dataset mode. batch-insights run gets groups but no examples. The new test asserts what the router's own rendering tests cannot: that every flag on these commands carries a group, that the group comes from the shared vocabulary, that a heading's flags are declared contiguously, and that no example names a flag its command does not declare. --- .../eval/batch-evaluation/evaluate/index.tsx | 60 +++++++++++- .../eval/batch-evaluation/simulate/index.tsx | 93 ++++++++++++++++-- .../eval/batch-insights/run/index.tsx | 18 +++- src/handlers/eval/helpGroups.test.ts | 97 +++++++++++++++++++ src/handlers/eval/helpGroups.tsx | 21 ++++ .../eval/online-eval/create/index.tsx | 81 +++++++++++++--- .../eval/online-eval/update/index.tsx | 81 ++++++++++++---- src/handlers/eval/sessionSource.tsx | 41 +++++--- 8 files changed, 423 insertions(+), 69 deletions(-) create mode 100644 src/handlers/eval/helpGroups.test.ts create mode 100644 src/handlers/eval/helpGroups.tsx diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index f6c73c512..82894d6ae 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -7,22 +7,74 @@ import type { Core } from "../../../types"; import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +import { HELP_GROUP } from "../../helpGroups"; export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ name: "evaluate", description: "evaluate existing sessions service-side (async; returns a job ID)", flags: [ + flag("name", "batch evaluation name (must be unique in the account)", z.string().optional(), { + group: HELP_GROUP.configuration, + }), + flag("description", "optional description", z.string().optional(), { + group: HELP_GROUP.configuration, + }), + flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { + group: HELP_GROUP.configuration, + }), ...SessionSource.flags, - flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { + group: HELP_GROUP.evaluation, + }), flag( "ground-truth", "session ground truth (JSON SessionMetadataShape[]; inline, file://, or -)", z.string().optional(), + { group: HELP_GROUP.evaluation }, ), - flag("name", "batch evaluation name (must be unique in the account)", z.string().optional()), - flag("description", "optional description", z.string().optional()), - flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), + ], + examples: [ + { + description: "Evaluate a Runtime with multiple evaluators", + command: [ + "agentcore eval batch-evaluation evaluate", + "--agent my-runtime", + "--evaluators Builtin.Helpfulness Builtin.Correctness", + "--name weekly-quality", + ], + }, + { + description: "Evaluate sessions within a UTC time window", + command: [ + "agentcore eval batch-evaluation evaluate", + "--agent my-runtime", + "--start-time 2026-09-01T00:00:00Z", + "--end-time 2026-09-08T00:00:00Z", + "--evaluators Builtin.Helpfulness", + "--name weekly-helpfulness", + ], + }, + { + description: "Evaluate specific sessions from a named Runtime endpoint", + command: [ + "agentcore eval batch-evaluation evaluate", + "--agent my-runtime", + "--endpoint BETA", + "--session-ids session-123 session-456", + "--evaluators Builtin.Correctness", + "--name selected-sessions", + ], + }, + { + description: "Evaluate sessions an online evaluation already sampled", + command: [ + "agentcore eval batch-evaluation evaluate", + "--online-eval online-eval-id", + "--evaluators Builtin.Helpfulness", + "--name sampled-sessions", + ], + }, ], handle: async (ctx, flags) => { if (!flags["name"]) { diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index a880a3ddf..8c271f01d 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -6,6 +6,7 @@ import type { AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; +import { HELP_GROUP } from "../../helpGroups"; // Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror // `runtime invoke`. @@ -14,34 +15,106 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => name: "simulate", description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions", flags: [ - flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional()), - flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), { + group: HELP_GROUP.runtimeInvocation, + }), + flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { + group: HELP_GROUP.runtimeInvocation, + }), flag( "payload-template", 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', z.string().optional(), + { group: HELP_GROUP.runtimeInvocation }, ), flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), { + group: HELP_GROUP.runtimeInvocation, sensitive: true, }), flag( "bearer-token", "CUSTOM_JWT bearer token (for JWT-auth Runtimes)", z.string().optional(), - { sensitive: true }, + { group: HELP_GROUP.runtimeInvocation, sensitive: true }, ), - flag("user-id", "Runtime user ID", z.string().optional()), - flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()), - flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()), - flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), - flag("name", "batch evaluation name (unique in the account)", z.string().optional()), - flag("description", "description for the batch evaluation", z.string().optional()), - flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), + flag("user-id", "Runtime user ID", z.string().optional(), { + group: HELP_GROUP.runtimeInvocation, + }), + flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional(), { + group: HELP_GROUP.dataset, + }), + flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), { + group: HELP_GROUP.dataset, + }), flag( "ingestion-wait-ms", "ms to wait for span ingestion before grading (default 180000; 0 to skip)", z.coerce.number().int().nonnegative().optional(), + { group: HELP_GROUP.dataset }, ), + flag("name", "batch evaluation name (unique in the account)", z.string().optional(), { + group: HELP_GROUP.configuration, + }), + flag("description", "description for the batch evaluation", z.string().optional(), { + group: HELP_GROUP.configuration, + }), + flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { + group: HELP_GROUP.configuration, + }), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { + group: HELP_GROUP.evaluation, + }), + ], + examples: [ + { + description: "Replay a local JSONL dataset and evaluate the generated sessions", + command: [ + "agentcore eval batch-evaluation simulate", + "--runtime-id my-runtime-id", + `--payload-template '{"prompt":"{input}"}'`, + "--dataset ./evaluation-dataset.jsonl", + "--evaluators Builtin.Helpfulness Builtin.Correctness", + "--name dataset-quality", + ], + }, + { + description: "Use a managed dataset version and a named Runtime endpoint", + command: [ + "agentcore eval batch-evaluation simulate", + "--runtime-id my-runtime-id", + "--qualifier BETA", + `--payload-template '{"prompt":"{input}"}'`, + "--dataset dataset-abc123", + "--dataset-version 2", + "--evaluators Builtin.Helpfulness", + "--name dataset-version-2", + ], + }, + { + description: "Invoke a CUSTOM_JWT Runtime", + command: [ + "agentcore eval batch-evaluation simulate", + "--runtime-id my-runtime-id", + `--payload-template '{"prompt":"{input}"}'`, + "--dataset ./evaluation-dataset.jsonl", + '--bearer-token "$RUNTIME_TOKEN"', + "--user-id user-123", + "--evaluators Builtin.Correctness", + "--name authenticated-simulation", + ], + }, + { + description: "Pass application headers to every invocation", + command: [ + "agentcore eval batch-evaluation simulate", + "--runtime-id my-runtime-id", + `--payload-template '{"prompt":"{input}"}'`, + "--dataset ./evaluation-dataset.jsonl", + '--header "X-Tenant-Id: customer-123" "X-Request-Source: evaluation"', + "--evaluators Builtin.Helpfulness", + "--name tenant-evaluation", + ], + }, ], handle: async (ctx, flags) => { if (!flags["runtime-id"]) diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx index 6d9d2a037..c57ed48a3 100644 --- a/src/handlers/eval/batch-insights/run/index.tsx +++ b/src/handlers/eval/batch-insights/run/index.tsx @@ -6,6 +6,7 @@ import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +import { HELP_GROUP } from "../../helpGroups"; const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis"; @@ -14,16 +15,25 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => name: "run", description: "start an asynchronous batch insights run over existing sessions", flags: [ + flag("name", "batch insights name (must be unique in the account)", z.string().optional(), { + group: HELP_GROUP.configuration, + }), + flag("description", "optional description", z.string().optional(), { + group: HELP_GROUP.configuration, + }), + flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional(), { + group: HELP_GROUP.configuration, + }), ...SessionSource.flags, - flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT])), + flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT]), { + group: HELP_GROUP.analysis, + }), flag( "evaluators", "optional evaluator ID(s) to run alongside the insights", z.array(z.string()).optional(), + { group: HELP_GROUP.analysis }, ), - flag("name", "batch insights name (must be unique in the account)", z.string().optional()), - flag("description", "optional description", z.string().optional()), - flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional()), ], handle: async (ctx, flags) => { if (!flags["name"]) { diff --git a/src/handlers/eval/helpGroups.test.ts b/src/handlers/eval/helpGroups.test.ts new file mode 100644 index 000000000..0c4f8b319 --- /dev/null +++ b/src/handlers/eval/helpGroups.test.ts @@ -0,0 +1,97 @@ +import { test, expect, describe } from "bun:test"; +import type { Handler } from "../../router"; +import { createRootHandler } from "../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../testing"; +import { TestGlobalConfigAccessor } from "../../testing/"; +import { HELP_GROUP } from "./helpGroups"; + +// The router proves that a Flag.group renders as a `--help` heading in +// declaration order (see router/router.test.ts). What these commands need +// proving instead is that they all speak the shared vocabulary: a flag left +// ungrouped falls into a leftover "Options:" block, and a hand-typed heading +// that drifts by a character silently splits into a second one. Both read as +// bugs, and neither fails any behavioral test. + +const HEADINGS = new Set(Object.values(HELP_GROUP)); + +// The eval commands converted to grouped help, by path from the root. +const GROUPED_COMMANDS = [ + ["eval", "batch-evaluation", "evaluate"], + ["eval", "batch-evaluation", "simulate"], + ["eval", "batch-insights", "run"], + ["eval", "online-eval", "create"], + ["eval", "online-eval", "update"], +] as const; + +// Commands that carry worked examples in `--help`. `batch-insights run` groups +// its flags but has no examples yet, so it is deliberately absent. +const COMMANDS_WITH_EXAMPLES = [ + ["eval", "batch-evaluation", "evaluate"], + ["eval", "batch-evaluation", "simulate"], + ["eval", "online-eval", "create"], + ["eval", "online-eval", "update"], +] as const; + +function resolve(path: readonly string[]): Handler { + const io = testIO(); + let node: Handler = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + for (const name of path) { + const child = node.children().find((c) => c.name() === name); + if (!child) throw new Error(`no '${name}' under '${node.name()}'`); + node = child; + } + return node; +} + +describe("eval help groups", () => { + for (const path of GROUPED_COMMANDS) { + const label = path.join(" "); + + test(`${label} groups every flag under a heading from the shared vocabulary`, () => { + const flags = resolve(path).flags(); + expect(flags.length).toBeGreaterThan(0); + + const ungrouped = flags.filter((f) => !f.group).map((f) => f.name); + expect(ungrouped).toEqual([]); + + const unknown = flags.map((f) => f.group!).filter((g) => !HEADINGS.has(g)); + expect(unknown).toEqual([]); + }); + + test(`${label} declares each heading's flags contiguously`, () => { + // Commander orders headings by the first flag declared in each, so a flag + // declared away from its heading-mates would render under the heading but + // reorder the headings themselves. + const groups = resolve(path) + .flags() + .map((f) => f.group!); + const firstSeen = [...new Set(groups)]; + expect(groups).toEqual(firstSeen.flatMap((g) => groups.filter((x) => x === g))); + }); + } + + for (const path of COMMANDS_WITH_EXAMPLES) { + const label = path.join(" "); + + test(`${label} carries examples that name only flags it declares`, () => { + const command = resolve(path); + const examples = command.examples?.() ?? []; + expect(examples.length).toBeGreaterThan(0); + + const declared = new Set(command.flags().map((f) => `--${f.name}`)); + for (const { description, command: invocation } of examples) { + expect(description).not.toEndWith(":"); + const text = Array.isArray(invocation) ? invocation.join(" ") : invocation; + expect(text).toStartWith(`agentcore ${label}`); + // Catches an example left behind by a flag rename. + for (const token of text.match(/--[a-z][a-z0-9-]*/g) ?? []) { + expect(declared).toContain(token); + } + } + }); + } +}); diff --git a/src/handlers/eval/helpGroups.tsx b/src/handlers/eval/helpGroups.tsx new file mode 100644 index 000000000..5903488e0 --- /dev/null +++ b/src/handlers/eval/helpGroups.tsx @@ -0,0 +1,21 @@ +// The shared `--help` group vocabulary for the eval commands. These commands +// have enough flags that a single flat option list is hard to skim, and they +// overlap heavily — batch evaluation, batch insights, and online evaluation all +// name a session source, narrow it, and apply evaluators. Naming the headings +// once means the same concept reads the same everywhere, and a typo cannot +// silently split one heading into two. +export const HELP_GROUP = { + runtimeInvocation: "Runtime invocation:", + dataset: "Dataset:", + target: "Target:", + configuration: "Configuration:", + // The exclusive spelling is for commands where a source must be chosen at + // creation; `sessionSource` is for update, where leaving it alone keeps the + // existing source. + sessionSourceExclusive: "Session source (choose exactly one):", + sessionSource: "Session source:", + sourceFilters: "Source filters:", + evaluation: "Evaluation:", + analysis: "Analysis:", + execution: "Execution:", +} as const; diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index d825e1001..f154873e7 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -6,55 +6,106 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { HELP_GROUP } from "../../helpGroups"; export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "create", description: "create an online evaluation config", flags: [ - flag("name", "the name of the online evaluation config", z.string().optional()), - flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional()), + flag("name", "the name of the online evaluation config", z.string().optional(), { + group: HELP_GROUP.configuration, + }), flag( - "endpoint", - "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", + "description", + "a description of the config's monitoring purpose", z.string().optional(), + { + group: HELP_GROUP.configuration, + }, ), + flag( + "enable-on-create", + "whether to enable evaluation immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + { group: HELP_GROUP.configuration }, + ), + flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { + group: HELP_GROUP.sessionSourceExclusive, + }), flag( "data-source-config", "the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin), as an alternative to --agent", z.string().optional(), + { group: HELP_GROUP.sessionSourceExclusive }, ), - flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()), + flag( + "endpoint", + "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", + z.string().optional(), + { group: HELP_GROUP.sessionSourceExclusive }, + ), + flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional(), { + group: HELP_GROUP.evaluation, + }), flag( "sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().min(0.01).max(100).optional(), + { group: HELP_GROUP.evaluation }, ), flag( "session-timeout-minutes", "minutes of inactivity before a session is considered complete (1-1440, default 15)", z.number().int().min(1).max(1440).optional(), + { group: HELP_GROUP.evaluation }, ), flag( "filters", "trace filters (JSON Filter[]; inline, file://, or - for stdin)", z.string().optional(), + { group: HELP_GROUP.evaluation }, ), flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", z.string().optional(), + { group: HELP_GROUP.execution }, ), - flag( - "enable-on-create", - "whether to enable evaluation immediately (default true; pass false to create it paused)", - z.enum(["true", "false"]).optional(), - ), - flag( - "description", - "a description of the config's monitoring purpose", - z.string().optional(), - ), + ], + examples: [ + { + description: "Sample 10% of a Runtime's traffic with two evaluators", + command: [ + "agentcore eval online-eval create", + "--name production-quality", + "--agent my-runtime", + "--sampling-rate 10", + "--evaluators Builtin.Helpfulness Builtin.Correctness", + ], + }, + { + description: "Sample traces from a raw DataSourceConfig held in a file", + command: [ + "agentcore eval online-eval create", + "--name source-results", + "--data-source-config file://data-source-config.json", + "--sampling-rate 25", + "--evaluators Builtin.Helpfulness", + ], + }, + { + description: "Create the config paused, to enable later with `resume`", + command: [ + "agentcore eval online-eval create", + "--name staged-quality", + "--agent my-runtime", + "--endpoint BETA", + "--sampling-rate 5", + "--evaluators Builtin.Correctness", + "--enable-on-create false", + ], + }, ], handle: async (ctx, flags) => { if (!flags["name"]) diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 664f78470..68c68b29f 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -7,56 +7,97 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { HELP_GROUP } from "../../helpGroups"; export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "update", description: "update an online evaluation config", flags: [ - flag("id", "the ID of the online evaluation config to update", z.string().optional()), + flag("id", "the ID of the online evaluation config to update", z.string().optional(), { + group: HELP_GROUP.target, + }), + flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { + group: HELP_GROUP.sessionSource, + }), flag( - "sampling-rate", - "percentage of sessions to sample (0.01-100)", - z.number().min(0.01).max(100).optional(), + "data-source-config", + "replace the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin)", + z.string().optional(), + { group: HELP_GROUP.sessionSource }, ), flag( - "session-timeout-minutes", - "minutes of inactivity before a session is considered complete (1-1440)", - z.number().int().min(1).max(1440).optional(), + "endpoint", + "re-scope monitoring to a different agent endpoint qualifier", + z.string().optional(), + { group: HELP_GROUP.sourceFilters }, ), flag( - "filters", - "trace filters (JSON Filter[]; inline, file://, or - for stdin)", - z.string().optional(), + "clear-endpoint", + "reset the endpoint scope to the default qualifier (pass true)", + z.enum(["true", "false"]).optional(), + { group: HELP_GROUP.sourceFilters }, ), flag( "evaluators", "the ID(s) of the evaluators to apply (replaces the existing list)", z.array(z.string()).optional(), + { group: HELP_GROUP.evaluation }, ), - flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional()), flag( - "endpoint", - "re-scope monitoring to a different agent endpoint qualifier", - z.string().optional(), + "sampling-rate", + "percentage of sessions to sample (0.01-100)", + z.number().min(0.01).max(100).optional(), + { group: HELP_GROUP.evaluation }, ), flag( - "clear-endpoint", - "reset the endpoint scope to the default qualifier (pass true)", - z.enum(["true", "false"]).optional(), + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete (1-1440)", + z.number().int().min(1).max(1440).optional(), + { group: HELP_GROUP.evaluation }, ), flag( - "data-source-config", - "replace the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin)", + "filters", + "trace filters (JSON Filter[]; inline, file://, or - for stdin)", z.string().optional(), + { group: HELP_GROUP.evaluation }, + ), + flag( + "role-arn", + "replace the IAM role the online evaluation assumes", + z.string().optional(), + { group: HELP_GROUP.execution }, ), - flag("role-arn", "replace the IAM role the online evaluation assumes", z.string().optional()), flag( "update-role", "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", z.enum(["true", "false"]).optional(), + { group: HELP_GROUP.execution }, ), ], + examples: [ + { + description: "Change the evaluators and the sampling rate", + command: [ + "agentcore eval online-eval update", + "--id online-eval-123", + "--sampling-rate 20", + "--evaluators Builtin.Helpfulness Builtin.Correctness", + ], + }, + { + description: "Re-scope monitoring to a different Runtime endpoint", + command: ["agentcore eval online-eval update", "--id online-eval-123", "--endpoint BETA"], + }, + { + description: "Bring your own execution role instead of the provisioned one", + command: [ + "agentcore eval online-eval update", + "--id online-eval-123", + "--role-arn arn:aws:iam::123456789012:role/MyOnlineEvalRole", + ], + }, + ], handle: async (ctx, flags) => { if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); if (flags["endpoint"] && flags["clear-endpoint"] === "true") { diff --git a/src/handlers/eval/sessionSource.tsx b/src/handlers/eval/sessionSource.tsx index aa8facd96..44fc64cb5 100644 --- a/src/handlers/eval/sessionSource.tsx +++ b/src/handlers/eval/sessionSource.tsx @@ -3,41 +3,50 @@ import z from "zod"; import { InputValidationError } from "../../errors"; import { SourceResolver, type AppIO } from "../../io"; import { flag, type Flag } from "../../router"; +import { HELP_GROUP } from "./helpGroups"; import { assertMutuallyExclusiveFlags, parseJsonFlag } from "../utils"; import type { SessionSourceValue, SessionWindow } from "./types"; export class SessionSource { + // Declared source-arms first so `--help` lists the source heading above the + // filter heading: Commander orders headings by the first flag declared in each. + // The group names carry what the descriptions used to have to say ("source:", + // "filter:"), so those prefixes are gone. static readonly flags = [ - flag("agent", "source: harness ID or Runtime ID whose sessions to use", z.string().optional()), - flag( - "endpoint", - "Runtime endpoint qualifier (default DEFAULT; only with --agent)", - z.string().optional(), - ), + flag("agent", "harness ID or Runtime ID whose sessions to use", z.string().optional(), { + group: HELP_GROUP.sessionSourceExclusive, + }), flag( "online-eval", - "source: use sessions an online-eval config already sampled", + "use sessions an online-eval config already sampled", z.string().optional(), + { group: HELP_GROUP.sessionSourceExclusive }, ), flag( "data-source-config", - "source: raw DataSourceConfig JSON (inline, file://, or -); escape hatch", + "raw DataSourceConfig JSON (inline, file://, or -); escape hatch", z.string().optional(), + { group: HELP_GROUP.sessionSourceExclusive }, ), flag( - "start-time", - "time filter: window start (ISO-8601, with --end-time)", - z.string().optional(), - ), - flag( - "end-time", - "time filter: window end (ISO-8601, with --start-time)", + "endpoint", + "Runtime endpoint qualifier (default DEFAULT; only with --agent)", z.string().optional(), + { group: HELP_GROUP.sourceFilters }, ), + flag("start-time", "window start (ISO-8601, with --end-time)", z.string().optional(), { + group: HELP_GROUP.sourceFilters, + }), + flag("end-time", "window end (ISO-8601, with --start-time)", z.string().optional(), { + group: HELP_GROUP.sourceFilters, + }), flag( "session-ids", - "filter: specific session IDs (only with --agent)", + "specific session IDs (only with --agent)", z.array(z.string()).optional(), + { + group: HELP_GROUP.sourceFilters, + }, ), ] as const; From 4d9616b9bde4473bfba642aa11e8c8f400f7f498 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 10 Sep 2026 22:03:01 +0000 Subject: [PATCH 3/5] refactor(eval): drop the examples mechanism and the shared group vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on #2262: - Each handler names its own `--help` headings; the shared HELP_GROUP module is gone, along with its unit test. - The Examples mechanism is removed entirely — Handler.examples, formatExamples, commandExamples, and the router tests that covered them. Worked invocations now live in each flag's `help:` block, which `--help` renders under "Parameter details" the way `harness create` already does. - Code comments added by the previous two commits are removed. Flags taking an API-shaped object gain that long-form documentation: --data-source-config (separately for the data-plane and control-plane shapes, which differ), --ground-truth, --filters, and --payload-template. Their one-line descriptions shrink to a type name, matching harness create. --- .../eval/batch-evaluation/evaluate/index.tsx | 79 ++++++--------- .../eval/batch-evaluation/simulate/index.tsx | 90 +++++------------ .../eval/batch-insights/run/index.tsx | 11 +-- src/handlers/eval/helpGroups.test.ts | 97 ------------------- src/handlers/eval/helpGroups.tsx | 21 ---- .../eval/online-eval/create/index.tsx | 69 ++++--------- .../eval/online-eval/dataSourceConfigHelp.tsx | 20 ++++ src/handlers/eval/online-eval/filtersHelp.tsx | 26 +++++ .../eval/online-eval/update/index.tsx | 58 ++++------- src/handlers/eval/sessionSource.tsx | 49 +++++++--- src/router/flags.tsx | 26 +---- src/router/handler.tsx | 27 ------ src/router/index.tsx | 2 - src/router/router.test.ts | 75 -------------- src/router/router.tsx | 32 +----- 15 files changed, 178 insertions(+), 504 deletions(-) delete mode 100644 src/handlers/eval/helpGroups.test.ts delete mode 100644 src/handlers/eval/helpGroups.tsx create mode 100644 src/handlers/eval/online-eval/dataSourceConfigHelp.tsx create mode 100644 src/handlers/eval/online-eval/filtersHelp.tsx diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index 82894d6ae..9e8fc9cd9 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -7,7 +7,30 @@ import type { Core } from "../../../types"; import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { SessionSource } from "../../sessionSource"; -import { HELP_GROUP } from "../../helpGroups"; + +const groundTruthHelp = `(JSON: list of objects) +Expected answers for the sessions being evaluated, so an evaluator can score a +response against a reference instead of judging it on its own. Each entry names +one session; omit an entry for a session that has no reference answer. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + [ + { + "sessionId": "string", // [required] the session the reference applies to + "testScenarioId": "string", // groups sessions replaying the same scenario + "groundTruth": { + "inline": "string" // the expected answer + } + }, + ... + ] + +Example: + --ground-truth '[{"sessionId":"session-123","groundTruth":{"inline":"The order shipped on Tuesday."}}]' + + --ground-truth file://ground-truth.json`; export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ @@ -15,67 +38,25 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => description: "evaluate existing sessions service-side (async; returns a job ID)", flags: [ flag("name", "batch evaluation name (must be unique in the account)", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag("description", "optional description", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), ...SessionSource.flags, flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { - group: HELP_GROUP.evaluation, + group: "Evaluation:", }), flag( "ground-truth", - "session ground truth (JSON SessionMetadataShape[]; inline, file://, or -)", + "expected answers for the sessions (JSON SessionMetadataShape[])", z.string().optional(), - { group: HELP_GROUP.evaluation }, + { group: "Evaluation:", help: groundTruthHelp }, ), ], - examples: [ - { - description: "Evaluate a Runtime with multiple evaluators", - command: [ - "agentcore eval batch-evaluation evaluate", - "--agent my-runtime", - "--evaluators Builtin.Helpfulness Builtin.Correctness", - "--name weekly-quality", - ], - }, - { - description: "Evaluate sessions within a UTC time window", - command: [ - "agentcore eval batch-evaluation evaluate", - "--agent my-runtime", - "--start-time 2026-09-01T00:00:00Z", - "--end-time 2026-09-08T00:00:00Z", - "--evaluators Builtin.Helpfulness", - "--name weekly-helpfulness", - ], - }, - { - description: "Evaluate specific sessions from a named Runtime endpoint", - command: [ - "agentcore eval batch-evaluation evaluate", - "--agent my-runtime", - "--endpoint BETA", - "--session-ids session-123 session-456", - "--evaluators Builtin.Correctness", - "--name selected-sessions", - ], - }, - { - description: "Evaluate sessions an online evaluation already sampled", - command: [ - "agentcore eval batch-evaluation evaluate", - "--online-eval online-eval-id", - "--evaluators Builtin.Helpfulness", - "--name sampled-sessions", - ], - }, - ], handle: async (ctx, flags) => { if (!flags["name"]) { throw new InputValidationError("required option '--name ' not specified"); diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index 8c271f01d..32ca14728 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -6,7 +6,16 @@ import type { AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; -import { HELP_GROUP } from "../../helpGroups"; + +const payloadTemplateHelp = `(JSON object) +The request body sent to the Runtime for each dataset example. Every occurrence +of {input} is replaced with that example's input, so the template describes the +shape your agent expects and {input} marks where the prompt goes. + +Example: + --payload-template '{"prompt":"{input}"}' + + --payload-template '{"messages":[{"role":"user","content":"{input}"}],"stream":false}'`; // Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror // `runtime invoke`. @@ -16,106 +25,55 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions", flags: [ flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), { - group: HELP_GROUP.runtimeInvocation, + group: "Runtime invocation:", }), flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { - group: HELP_GROUP.runtimeInvocation, + group: "Runtime invocation:", }), flag( "payload-template", - 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + "request body per example (JSON object); {input} is replaced with the input", z.string().optional(), - { group: HELP_GROUP.runtimeInvocation }, + { group: "Runtime invocation:", help: payloadTemplateHelp }, ), flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), { - group: HELP_GROUP.runtimeInvocation, + group: "Runtime invocation:", sensitive: true, }), flag( "bearer-token", "CUSTOM_JWT bearer token (for JWT-auth Runtimes)", z.string().optional(), - { group: HELP_GROUP.runtimeInvocation, sensitive: true }, + { group: "Runtime invocation:", sensitive: true }, ), flag("user-id", "Runtime user ID", z.string().optional(), { - group: HELP_GROUP.runtimeInvocation, + group: "Runtime invocation:", }), flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional(), { - group: HELP_GROUP.dataset, + group: "Dataset:", }), flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), { - group: HELP_GROUP.dataset, + group: "Dataset:", }), flag( "ingestion-wait-ms", "ms to wait for span ingestion before grading (default 180000; 0 to skip)", z.coerce.number().int().nonnegative().optional(), - { group: HELP_GROUP.dataset }, + { group: "Dataset:" }, ), flag("name", "batch evaluation name (unique in the account)", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag("description", "description for the batch evaluation", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { - group: HELP_GROUP.evaluation, + group: "Evaluation:", }), ], - examples: [ - { - description: "Replay a local JSONL dataset and evaluate the generated sessions", - command: [ - "agentcore eval batch-evaluation simulate", - "--runtime-id my-runtime-id", - `--payload-template '{"prompt":"{input}"}'`, - "--dataset ./evaluation-dataset.jsonl", - "--evaluators Builtin.Helpfulness Builtin.Correctness", - "--name dataset-quality", - ], - }, - { - description: "Use a managed dataset version and a named Runtime endpoint", - command: [ - "agentcore eval batch-evaluation simulate", - "--runtime-id my-runtime-id", - "--qualifier BETA", - `--payload-template '{"prompt":"{input}"}'`, - "--dataset dataset-abc123", - "--dataset-version 2", - "--evaluators Builtin.Helpfulness", - "--name dataset-version-2", - ], - }, - { - description: "Invoke a CUSTOM_JWT Runtime", - command: [ - "agentcore eval batch-evaluation simulate", - "--runtime-id my-runtime-id", - `--payload-template '{"prompt":"{input}"}'`, - "--dataset ./evaluation-dataset.jsonl", - '--bearer-token "$RUNTIME_TOKEN"', - "--user-id user-123", - "--evaluators Builtin.Correctness", - "--name authenticated-simulation", - ], - }, - { - description: "Pass application headers to every invocation", - command: [ - "agentcore eval batch-evaluation simulate", - "--runtime-id my-runtime-id", - `--payload-template '{"prompt":"{input}"}'`, - "--dataset ./evaluation-dataset.jsonl", - '--header "X-Tenant-Id: customer-123" "X-Request-Source: evaluation"', - "--evaluators Builtin.Helpfulness", - "--name tenant-evaluation", - ], - }, - ], handle: async (ctx, flags) => { if (!flags["runtime-id"]) throw new InputValidationError("required option '--runtime-id' not specified"); diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx index c57ed48a3..b93adb1ef 100644 --- a/src/handlers/eval/batch-insights/run/index.tsx +++ b/src/handlers/eval/batch-insights/run/index.tsx @@ -6,7 +6,6 @@ import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { SessionSource } from "../../sessionSource"; -import { HELP_GROUP } from "../../helpGroups"; const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis"; @@ -16,23 +15,23 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => description: "start an asynchronous batch insights run over existing sessions", flags: [ flag("name", "batch insights name (must be unique in the account)", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag("description", "optional description", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), ...SessionSource.flags, flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT]), { - group: HELP_GROUP.analysis, + group: "Analysis:", }), flag( "evaluators", "optional evaluator ID(s) to run alongside the insights", z.array(z.string()).optional(), - { group: HELP_GROUP.analysis }, + { group: "Analysis:" }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/helpGroups.test.ts b/src/handlers/eval/helpGroups.test.ts deleted file mode 100644 index 0c4f8b319..000000000 --- a/src/handlers/eval/helpGroups.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { test, expect, describe } from "bun:test"; -import type { Handler } from "../../router"; -import { createRootHandler } from "../index"; -import { createSilentLogger, TestCoreClient, testIO } from "../../testing"; -import { TestGlobalConfigAccessor } from "../../testing/"; -import { HELP_GROUP } from "./helpGroups"; - -// The router proves that a Flag.group renders as a `--help` heading in -// declaration order (see router/router.test.ts). What these commands need -// proving instead is that they all speak the shared vocabulary: a flag left -// ungrouped falls into a leftover "Options:" block, and a hand-typed heading -// that drifts by a character silently splits into a second one. Both read as -// bugs, and neither fails any behavioral test. - -const HEADINGS = new Set(Object.values(HELP_GROUP)); - -// The eval commands converted to grouped help, by path from the root. -const GROUPED_COMMANDS = [ - ["eval", "batch-evaluation", "evaluate"], - ["eval", "batch-evaluation", "simulate"], - ["eval", "batch-insights", "run"], - ["eval", "online-eval", "create"], - ["eval", "online-eval", "update"], -] as const; - -// Commands that carry worked examples in `--help`. `batch-insights run` groups -// its flags but has no examples yet, so it is deliberately absent. -const COMMANDS_WITH_EXAMPLES = [ - ["eval", "batch-evaluation", "evaluate"], - ["eval", "batch-evaluation", "simulate"], - ["eval", "online-eval", "create"], - ["eval", "online-eval", "update"], -] as const; - -function resolve(path: readonly string[]): Handler { - const io = testIO(); - let node: Handler = createRootHandler(new TestCoreClient(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - for (const name of path) { - const child = node.children().find((c) => c.name() === name); - if (!child) throw new Error(`no '${name}' under '${node.name()}'`); - node = child; - } - return node; -} - -describe("eval help groups", () => { - for (const path of GROUPED_COMMANDS) { - const label = path.join(" "); - - test(`${label} groups every flag under a heading from the shared vocabulary`, () => { - const flags = resolve(path).flags(); - expect(flags.length).toBeGreaterThan(0); - - const ungrouped = flags.filter((f) => !f.group).map((f) => f.name); - expect(ungrouped).toEqual([]); - - const unknown = flags.map((f) => f.group!).filter((g) => !HEADINGS.has(g)); - expect(unknown).toEqual([]); - }); - - test(`${label} declares each heading's flags contiguously`, () => { - // Commander orders headings by the first flag declared in each, so a flag - // declared away from its heading-mates would render under the heading but - // reorder the headings themselves. - const groups = resolve(path) - .flags() - .map((f) => f.group!); - const firstSeen = [...new Set(groups)]; - expect(groups).toEqual(firstSeen.flatMap((g) => groups.filter((x) => x === g))); - }); - } - - for (const path of COMMANDS_WITH_EXAMPLES) { - const label = path.join(" "); - - test(`${label} carries examples that name only flags it declares`, () => { - const command = resolve(path); - const examples = command.examples?.() ?? []; - expect(examples.length).toBeGreaterThan(0); - - const declared = new Set(command.flags().map((f) => `--${f.name}`)); - for (const { description, command: invocation } of examples) { - expect(description).not.toEndWith(":"); - const text = Array.isArray(invocation) ? invocation.join(" ") : invocation; - expect(text).toStartWith(`agentcore ${label}`); - // Catches an example left behind by a flag rename. - for (const token of text.match(/--[a-z][a-z0-9-]*/g) ?? []) { - expect(declared).toContain(token); - } - } - }); - } -}); diff --git a/src/handlers/eval/helpGroups.tsx b/src/handlers/eval/helpGroups.tsx deleted file mode 100644 index 5903488e0..000000000 --- a/src/handlers/eval/helpGroups.tsx +++ /dev/null @@ -1,21 +0,0 @@ -// The shared `--help` group vocabulary for the eval commands. These commands -// have enough flags that a single flat option list is hard to skim, and they -// overlap heavily — batch evaluation, batch insights, and online evaluation all -// name a session source, narrow it, and apply evaluators. Naming the headings -// once means the same concept reads the same everywhere, and a typo cannot -// silently split one heading into two. -export const HELP_GROUP = { - runtimeInvocation: "Runtime invocation:", - dataset: "Dataset:", - target: "Target:", - configuration: "Configuration:", - // The exclusive spelling is for commands where a source must be chosen at - // creation; `sessionSource` is for update, where leaving it alone keeps the - // existing source. - sessionSourceExclusive: "Session source (choose exactly one):", - sessionSource: "Session source:", - sourceFilters: "Source filters:", - evaluation: "Evaluation:", - analysis: "Analysis:", - execution: "Execution:", -} as const; diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index f154873e7..1ebdb5ab2 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -6,7 +6,8 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { HELP_GROUP } from "../../helpGroups"; +import { filtersHelp } from "../filtersHelp"; +import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ @@ -14,99 +15,63 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => description: "create an online evaluation config", flags: [ flag("name", "the name of the online evaluation config", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }), flag( "description", "a description of the config's monitoring purpose", z.string().optional(), { - group: HELP_GROUP.configuration, + group: "Configuration:", }, ), flag( "enable-on-create", "whether to enable evaluation immediately (default true; pass false to create it paused)", z.enum(["true", "false"]).optional(), - { group: HELP_GROUP.configuration }, + { group: "Configuration:" }, ), flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { - group: HELP_GROUP.sessionSourceExclusive, + group: "Session source (choose exactly one):", }), flag( "data-source-config", - "the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin), as an alternative to --agent", + "the traces to sample (JSON DataSourceConfig), as an alternative to --agent", z.string().optional(), - { group: HELP_GROUP.sessionSourceExclusive }, + { group: "Session source (choose exactly one):", help: onlineEvalDataSourceConfigHelp }, ), flag( "endpoint", "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", z.string().optional(), - { group: HELP_GROUP.sessionSourceExclusive }, + { group: "Session source (choose exactly one):" }, ), flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional(), { - group: HELP_GROUP.evaluation, + group: "Evaluation:", }), flag( "sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().min(0.01).max(100).optional(), - { group: HELP_GROUP.evaluation }, + { group: "Evaluation:" }, ), flag( "session-timeout-minutes", "minutes of inactivity before a session is considered complete (1-1440, default 15)", z.number().int().min(1).max(1440).optional(), - { group: HELP_GROUP.evaluation }, - ), - flag( - "filters", - "trace filters (JSON Filter[]; inline, file://, or - for stdin)", - z.string().optional(), - { group: HELP_GROUP.evaluation }, + { group: "Evaluation:" }, ), + flag("filters", "trace filters (JSON Filter[])", z.string().optional(), { + group: "Evaluation:", + help: filtersHelp, + }), flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", z.string().optional(), - { group: HELP_GROUP.execution }, + { group: "Execution:" }, ), ], - examples: [ - { - description: "Sample 10% of a Runtime's traffic with two evaluators", - command: [ - "agentcore eval online-eval create", - "--name production-quality", - "--agent my-runtime", - "--sampling-rate 10", - "--evaluators Builtin.Helpfulness Builtin.Correctness", - ], - }, - { - description: "Sample traces from a raw DataSourceConfig held in a file", - command: [ - "agentcore eval online-eval create", - "--name source-results", - "--data-source-config file://data-source-config.json", - "--sampling-rate 25", - "--evaluators Builtin.Helpfulness", - ], - }, - { - description: "Create the config paused, to enable later with `resume`", - command: [ - "agentcore eval online-eval create", - "--name staged-quality", - "--agent my-runtime", - "--endpoint BETA", - "--sampling-rate 5", - "--evaluators Builtin.Correctness", - "--enable-on-create false", - ], - }, - ], handle: async (ctx, flags) => { if (!flags["name"]) throw new InputValidationError("required option '--name ' not specified"); diff --git a/src/handlers/eval/online-eval/dataSourceConfigHelp.tsx b/src/handlers/eval/online-eval/dataSourceConfigHelp.tsx new file mode 100644 index 000000000..e079181fc --- /dev/null +++ b/src/handlers/eval/online-eval/dataSourceConfigHelp.tsx @@ -0,0 +1,20 @@ +export const onlineEvalDataSourceConfigHelp = `(JSON: tagged union object) +Which traces are sampled, for sources the --agent convenience flag cannot +express. Only top-level key: cloudWatchLogs. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchLogs": { + "serviceNames": ["string", ...], // [required] e.g. "my_agent.DEFAULT" + "logGroupNames": ["string", ...], // exact group names + "logGroupNamePrefixes": ["string", ...] // or match by prefix instead + } // supply one of the two name lists + } + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore-control/latest/APIReference/API_DataSourceConfig.html + +Example: + --data-source-config '{"cloudWatchLogs":{"logGroupNames":["/aws/bedrock-agentcore/runtimes/my-runtime-DEFAULT"],"serviceNames":["my_agent.DEFAULT"]}}'`; diff --git a/src/handlers/eval/online-eval/filtersHelp.tsx b/src/handlers/eval/online-eval/filtersHelp.tsx new file mode 100644 index 000000000..35e67aa87 --- /dev/null +++ b/src/handlers/eval/online-eval/filtersHelp.tsx @@ -0,0 +1,26 @@ +export const filtersHelp = `(JSON: list of objects) +Narrows which sampled traces are evaluated. A trace must match every filter in +the list. Omit it to evaluate every sampled trace. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + [ + { + "key": "string", // [required] trace field to filter on + "operator": "Equals" | "NotEquals" | "Contains" | "NotContains" + | "GreaterThan" | "GreaterThanOrEqual" + | "LessThan" | "LessThanOrEqual", // [required] + "value": { // [required] exactly one key + "stringValue": "string", + "doubleValue": number, + "booleanValue": true | false + } + }, + ... + ] + +Example: + --filters '[{"key":"attributes.customer_tier","operator":"Equals","value":{"stringValue":"enterprise"}}]' + + --filters file://filters.json`; diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 68c68b29f..f5d8e895e 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -7,7 +7,8 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { HELP_GROUP } from "../../helpGroups"; +import { filtersHelp } from "../filtersHelp"; +import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ @@ -15,89 +16,64 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => description: "update an online evaluation config", flags: [ flag("id", "the ID of the online evaluation config to update", z.string().optional(), { - group: HELP_GROUP.target, + group: "Target:", }), flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { - group: HELP_GROUP.sessionSource, + group: "Session source:", }), flag( "data-source-config", - "replace the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin)", + "replace the traces to sample (JSON DataSourceConfig)", z.string().optional(), - { group: HELP_GROUP.sessionSource }, + { group: "Session source:", help: onlineEvalDataSourceConfigHelp }, ), flag( "endpoint", "re-scope monitoring to a different agent endpoint qualifier", z.string().optional(), - { group: HELP_GROUP.sourceFilters }, + { group: "Source filters:" }, ), flag( "clear-endpoint", "reset the endpoint scope to the default qualifier (pass true)", z.enum(["true", "false"]).optional(), - { group: HELP_GROUP.sourceFilters }, + { group: "Source filters:" }, ), flag( "evaluators", "the ID(s) of the evaluators to apply (replaces the existing list)", z.array(z.string()).optional(), - { group: HELP_GROUP.evaluation }, + { group: "Evaluation:" }, ), flag( "sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().min(0.01).max(100).optional(), - { group: HELP_GROUP.evaluation }, + { group: "Evaluation:" }, ), flag( "session-timeout-minutes", "minutes of inactivity before a session is considered complete (1-1440)", z.number().int().min(1).max(1440).optional(), - { group: HELP_GROUP.evaluation }, - ), - flag( - "filters", - "trace filters (JSON Filter[]; inline, file://, or - for stdin)", - z.string().optional(), - { group: HELP_GROUP.evaluation }, + { group: "Evaluation:" }, ), + flag("filters", "replace the trace filters (JSON Filter[])", z.string().optional(), { + group: "Evaluation:", + help: filtersHelp, + }), flag( "role-arn", "replace the IAM role the online evaluation assumes", z.string().optional(), - { group: HELP_GROUP.execution }, + { group: "Execution:" }, ), flag( "update-role", "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", z.enum(["true", "false"]).optional(), - { group: HELP_GROUP.execution }, + { group: "Execution:" }, ), ], - examples: [ - { - description: "Change the evaluators and the sampling rate", - command: [ - "agentcore eval online-eval update", - "--id online-eval-123", - "--sampling-rate 20", - "--evaluators Builtin.Helpfulness Builtin.Correctness", - ], - }, - { - description: "Re-scope monitoring to a different Runtime endpoint", - command: ["agentcore eval online-eval update", "--id online-eval-123", "--endpoint BETA"], - }, - { - description: "Bring your own execution role instead of the provisioned one", - command: [ - "agentcore eval online-eval update", - "--id online-eval-123", - "--role-arn arn:aws:iam::123456789012:role/MyOnlineEvalRole", - ], - }, - ], handle: async (ctx, flags) => { if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); if (flags["endpoint"] && flags["clear-endpoint"] === "true") { diff --git a/src/handlers/eval/sessionSource.tsx b/src/handlers/eval/sessionSource.tsx index 44fc64cb5..b69c6d76a 100644 --- a/src/handlers/eval/sessionSource.tsx +++ b/src/handlers/eval/sessionSource.tsx @@ -3,49 +3,72 @@ import z from "zod"; import { InputValidationError } from "../../errors"; import { SourceResolver, type AppIO } from "../../io"; import { flag, type Flag } from "../../router"; -import { HELP_GROUP } from "./helpGroups"; import { assertMutuallyExclusiveFlags, parseJsonFlag } from "../utils"; import type { SessionSourceValue, SessionWindow } from "./types"; +const dataSourceConfigHelp = `(JSON: tagged union object) +Where sessions and traces are read from, for sources the --agent and +--online-eval convenience flags cannot express. Only top-level key: +cloudWatchLogs. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchLogs": { + "logGroupNames": ["string", ...], // [required] groups holding the traces + "serviceNames": ["string", ...], // e.g. "my_agent.DEFAULT" + "filterConfig": { + "sessionIds": ["string", ...], + "sessionFilterConfig": { + "startTime": "timestamp", + "endTime": "timestamp" + } + } + } + } + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_DataSourceConfig.html + +Example: + --data-source-config '{"cloudWatchLogs":{"logGroupNames":["/aws/bedrock-agentcore/runtimes/support_agent-AbC123XyZ9-DEFAULT"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionIds":["session-123"]}}}'`; + export class SessionSource { - // Declared source-arms first so `--help` lists the source heading above the - // filter heading: Commander orders headings by the first flag declared in each. - // The group names carry what the descriptions used to have to say ("source:", - // "filter:"), so those prefixes are gone. static readonly flags = [ flag("agent", "harness ID or Runtime ID whose sessions to use", z.string().optional(), { - group: HELP_GROUP.sessionSourceExclusive, + group: "Session source (choose exactly one):", }), flag( "online-eval", "use sessions an online-eval config already sampled", z.string().optional(), - { group: HELP_GROUP.sessionSourceExclusive }, + { group: "Session source (choose exactly one):" }, ), flag( "data-source-config", - "raw DataSourceConfig JSON (inline, file://, or -); escape hatch", + "the traces to read (JSON DataSourceConfig); escape hatch", z.string().optional(), - { group: HELP_GROUP.sessionSourceExclusive }, + { group: "Session source (choose exactly one):", help: dataSourceConfigHelp }, ), flag( "endpoint", "Runtime endpoint qualifier (default DEFAULT; only with --agent)", z.string().optional(), - { group: HELP_GROUP.sourceFilters }, + { group: "Source filters:" }, ), flag("start-time", "window start (ISO-8601, with --end-time)", z.string().optional(), { - group: HELP_GROUP.sourceFilters, + group: "Source filters:", }), flag("end-time", "window end (ISO-8601, with --start-time)", z.string().optional(), { - group: HELP_GROUP.sourceFilters, + group: "Source filters:", }), flag( "session-ids", "specific session IDs (only with --agent)", z.array(z.string()).optional(), { - group: HELP_GROUP.sourceFilters, + group: "Source filters:", }, ), ] as const; diff --git a/src/router/flags.tsx b/src/router/flags.tsx index 72f8945e6..1f59770a1 100644 --- a/src/router/flags.tsx +++ b/src/router/flags.tsx @@ -1,7 +1,7 @@ import { Option } from "commander"; import { InputValidationError } from "../errors"; import type { Context } from "./context"; -import type { Example, Flag, GlobalFlag } from "./handler"; +import type { Flag, GlobalFlag } from "./handler"; import { coerce, formatZodError, inspect } from "./schema"; // toOption builds a Commander Option from a flag's schema. A boolean that defaults @@ -56,30 +56,6 @@ export function formatParameterDetails(flags: Flag[]): string | undefined { return `\nParameter details:\n\n${sections.join("\n\n")}\n`; } -// Indentation of a rendered example: the description sits one level in, the -// command another, and a continued command line one level deeper again so the -// backslash-joined flags read as belonging to the line above. -const EXAMPLE_DESCRIPTION_INDENT = " "; -const EXAMPLE_COMMAND_INDENT = " "; -const EXAMPLE_CONTINUATION_INDENT = " "; - -// formatExamples renders a command's worked invocations into the block appended -// after the option list (and after Parameter details, when present). Authors -// supply the shell command only; the layout lives here so every command's -// examples line up identically no matter who wrote them. -export function formatExamples(examples: readonly Example[]): string | undefined { - if (examples.length === 0) return undefined; - - const blocks = examples.map(({ description, command }) => { - const body = Array.isArray(command) - ? command.join(` \\\n${EXAMPLE_CONTINUATION_INDENT}`) - : command; - return `${EXAMPLE_DESCRIPTION_INDENT}${description}:\n\n${EXAMPLE_COMMAND_INDENT}${body}`; - }); - - return `\nExamples:\n\n${blocks.join("\n\n")}\n`; -} - // attributeName mirrors how Commander camelCases an option name into the key it // stores on the parsed options object (e.g. "harness-id" -> "harnessId"). function attributeName(name: string): string { diff --git a/src/router/handler.tsx b/src/router/handler.tsx index d634a7f0e..abb24834b 100644 --- a/src/router/handler.tsx +++ b/src/router/handler.tsx @@ -14,26 +14,11 @@ export interface Flag { // Its first line is the type annotation shown next to the flag name; the // remaining lines are the body — prose, JSON syntax, examples. help?: string; - // group is the `--help` heading this flag is listed under. Commands whose - // option list is long enough to skim past benefit from semantic headings - // ("Session source:", "Result output:") over one flat "Options:" block. - // Ungrouped flags stay in Commander's default section. group?: string; // sensitive flags are redacted from debug logs by the withLogging middleware. sensitive?: boolean; } -// Example is one worked invocation shown in a command's `--help`. `command` is -// authored as the shell command itself, with no indentation or line -// continuations: a string renders on one line, and an array renders joined by -// ` \` + newline. Splitting the array is how the author chooses where the breaks -// fall — usually one flag per element — so the renderer never has to guess at a -// terminal width, and the printed command still pastes into a shell verbatim. -export interface Example { - description: string; - command: string | string[]; -} - // GlobalFlag is a group-level flag that is *also* a typed ContextKey: declared on // a Router, its validated value is injected into the context under itself, so any // descendant handler can retrieve it type-safely via `ctx.value(theGlobalFlag)`. @@ -108,11 +93,6 @@ export interface Handler { arguments(): Argument[]; // Middleware must preserve this metadata when wrapping a handler. doesSupportTui(): boolean; - // examples are the worked invocations appended to `--help` after the option - // list. Optional because only the handler that authors examples needs to - // answer it: compile() reads it off the authored node, so the middleware - // wrappers (which only forward `handle`) never have to carry it. - examples?(): readonly Example[] | undefined; // At runtime `handle` receives the validated, coerced flags object. The precise // shape is supplied to authors via createHandler's generic; the interface keeps // it erased so middleware can forward it uniformly. @@ -128,7 +108,6 @@ type CreateHandlerInput< description: string; flags?: F; arguments?: A; - examples?: readonly Example[]; handle?: HandleFn; children?: Handler[]; }; @@ -140,7 +119,6 @@ class BaseHandler implements Handler { _description: string; _flags: Flag[]; _arguments: Argument[]; - _examples?: readonly Example[]; _handle: HandleFn; _children: Handler[]; @@ -151,7 +129,6 @@ class BaseHandler implements Handler { this._description = input.description; this._flags = (input.flags ?? []) as Flag[]; this._arguments = (input.arguments ?? []) as Argument[]; - this._examples = input.examples; this._handle = (input.handle ?? noOpHandler) as HandleFn; this._children = input.children ?? []; } @@ -176,10 +153,6 @@ class BaseHandler implements Handler { return true; } - examples(): readonly Example[] | undefined { - return this._examples; - } - async handle(ctx: Context, flags: any, args: any): Promise { await this._handle(ctx, flags, args); } diff --git a/src/router/index.tsx b/src/router/index.tsx index ad31dca93..f1c6b78a4 100644 --- a/src/router/index.tsx +++ b/src/router/index.tsx @@ -13,11 +13,9 @@ export { isDefaultHandlerProvider, isTuiCommandSupported, commandParameterDetails, - commandExamples, } from "./router"; export { type Handler, - type Example, type Flag, type GlobalFlag, type Argument, diff --git a/src/router/router.test.ts b/src/router/router.test.ts index 31d04f155..9808ed240 100644 --- a/src/router/router.test.ts +++ b/src/router/router.test.ts @@ -919,13 +919,10 @@ test("grouped flags render under their headings, ungrouped ones under Options", expect(out).toContain("Session source:"); expect(out).toContain("Source filters:"); - // Headings appear in the order their first flag was declared, and each flag - // sits under its own heading rather than in one flat list. expect(out.indexOf("Session source:")).toBeLessThan(out.indexOf("Source filters:")); expect(out.indexOf("--agent")).toBeGreaterThan(out.indexOf("Session source:")); expect(out.indexOf("--start-time")).toBeGreaterThan(out.indexOf("Source filters:")); expect(out.indexOf("--end-time")).toBeGreaterThan(out.indexOf("Source filters:")); - // An ungrouped flag keeps Commander's default heading. expect(out.indexOf("--name")).toBeGreaterThan(out.indexOf("Options:")); }); @@ -960,75 +957,3 @@ test("a command without grouped flags leaves the generated help in Options", asy expect(out).not.toContain("Other options:"); expect(out).toContain("-h, --help"); }); - -test("handler examples render once, after the parameter details", async () => { - const create = createHandler({ - name: "create", - description: "", - flags: [ - flag("model", "model config (JSON)", z.string().optional(), { - help: `(JSON object)\nThe model configuration.`, - }), - ], - examples: [ - { description: "Create with a Bedrock model", command: `app create --model '{"a":1}'` }, - ], - handle: async () => {}, - }); - const root = new Router("app"); - root.handler(create); - - const out = await helpOutput(root, ["app", "create", "--help"]); - - expect(out).toContain("Examples:"); - expect(out).toContain(" Create with a Bedrock model:"); - expect(out).toContain(` app create --model '{"a":1}'`); - expect(out.indexOf("Examples:")).toBeGreaterThan(out.indexOf("Parameter details:")); - expect(out.split("Examples:").length - 1).toBe(1); -}); - -test("a multi-line example is joined with backslash continuations", async () => { - const evaluate = createHandler({ - name: "evaluate", - description: "", - examples: [ - { - description: "Evaluate a Runtime with two evaluators", - command: [ - "app evaluate", - "--agent my-runtime", - "--evaluators Builtin.Helpfulness Builtin.Correctness", - ], - }, - { description: "List what is already running", command: "app list" }, - ], - handle: async () => {}, - }); - const root = new Router("app"); - root.handler(evaluate); - - const out = await helpOutput(root, ["app", "evaluate", "--help"]); - - // The author supplies the shell command only; indentation and the trailing - // backslashes are the renderer's, so every command's examples line up. - expect(out).toContain( - [ - " Evaluate a Runtime with two evaluators:", - "", - " app evaluate \\", - " --agent my-runtime \\", - " --evaluators Builtin.Helpfulness Builtin.Correctness", - ].join("\n"), - ); - // A single-string command stays on one line. - expect(out).toContain(" List what is already running:\n\n app list"); -}); - -test("commands without examples have no Examples section", async () => { - const root = new Router("app"); - root.handler(leaf("get", () => {})); - - const out = await helpOutput(root, ["app", "get", "--help"]); - - expect(out).not.toContain("Examples:"); -}); diff --git a/src/router/router.tsx b/src/router/router.tsx index 26ee8d638..df938de28 100644 --- a/src/router/router.tsx +++ b/src/router/router.tsx @@ -1,13 +1,7 @@ import type { Argument, Flag, GlobalFlag, Handler } from "./handler"; import { type Middleware, type MiddlewareProvider, isMiddlewareProvider } from "./middleware"; import { type Context, type ContextKey, ValueContext, contextKey } from "./context"; -import { - applyGlobalFlags, - formatExamples, - formatParameterDetails, - parseFlags, - toOption, -} from "./flags"; +import { applyGlobalFlags, formatParameterDetails, parseFlags, toOption } from "./flags"; import { parseArguments, toCommanderArgument } from "./args"; import { Command, CommanderError, Option } from "commander"; @@ -54,16 +48,6 @@ export function commandParameterDetails(command: Command): string | undefined { : undefined; } -// commandExamples is the worked-invocation block `--help` appends after the -// parameter details; undefined when the command authors none. Same reason as -// commandParameterDetails for existing: added help text is absent from -// helpInformation(), so a TUI rendering has to ask for it separately. -export function commandExamples(command: Command): string | undefined { - if (!(command instanceof RoutedCommand)) return undefined; - const examples = command.handler.examples?.(); - return examples ? formatExamples(examples) : undefined; -} - interface TuiChildSupportProvider { supportsTuiCommand(commandName: string): boolean; } @@ -79,7 +63,6 @@ function withEffectiveTuiSupport(handler: Handler, supported: boolean): Handler flags: () => handler.flags(), arguments: () => handler.arguments(), doesSupportTui: () => supported, - examples: () => handler.examples?.(), handle: (ctx, flags, args) => handler.handle(ctx, flags, args), children: () => handler.children(), }; @@ -204,10 +187,6 @@ export function compile( declareFlags(c, ownFlags); declareArguments(c, node.arguments()); - // A command that groups its own flags would otherwise leave Commander's - // generated `-h, --help` alone in the default "Options:" heading, reading as - // if it were the command's only ungrouped option. Give it its own heading so - // every listed flag sits under a deliberate one. if (ownFlags.some((f) => f.group)) { c.addHelpOption( new Option("-h, --help", "display help for command").helpGroup("Other options:"), @@ -215,18 +194,11 @@ export function compile( } // Flags with long-form documentation get a "Parameter details" section after - // the option list in `--help` output. Examples follow it: Commander emits - // added help text in registration order, and the worked invocations read as - // the closing section. + // the option list in `--help` output. const parameterDetails = formatParameterDetails(ownFlags); if (parameterDetails) { c.addHelpText("after", parameterDetails); } - const examples = node.examples?.(); - const renderedExamples = examples ? formatExamples(examples) : undefined; - if (renderedExamples) { - c.addHelpText("after", renderedExamples); - } const own = isMiddlewareProvider(node) ? node.middlewares() : []; const nextStack = [...stack, ...own]; From c8936e93119add64e8276b2eca755d00abacf41b Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 10 Sep 2026 22:57:11 +0000 Subject: [PATCH 4/5] refactor(eval): name each command's help-group headings with consts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each handler declares the headings it uses once at the top of its own file rather than repeating the string at every flag — "Session source (choose exactly one):" was written three times in online-eval create alone. Declared in first-appearance order so reading them top-down matches the order the headings render. Nothing is shared between files: CONFIGURATION is declared independently in the four handlers that use it, and sessionSource owns SESSION_SOURCE and SOURCE_FILTERS because it declares those flags. `--help` output is byte-identical for all five commands. --- .../eval/batch-evaluation/evaluate/index.tsx | 13 +++++--- .../eval/batch-evaluation/simulate/index.tsx | 31 +++++++++++-------- .../eval/batch-insights/run/index.tsx | 13 +++++--- .../eval/online-eval/create/index.tsx | 27 +++++++++------- .../eval/online-eval/update/index.tsx | 28 ++++++++++------- src/handlers/eval/sessionSource.tsx | 17 +++++----- 6 files changed, 77 insertions(+), 52 deletions(-) diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index 9e8fc9cd9..8b95678ad 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -8,6 +8,9 @@ import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +const CONFIGURATION = "Configuration:"; +const EVALUATION = "Evaluation:"; + const groundTruthHelp = `(JSON: list of objects) Expected answers for the sessions being evaluated, so an evaluator can score a response against a reference instead of judging it on its own. Each entry names @@ -38,23 +41,23 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => description: "evaluate existing sessions service-side (async; returns a job ID)", flags: [ flag("name", "batch evaluation name (must be unique in the account)", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag("description", "optional description", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), ...SessionSource.flags, flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { - group: "Evaluation:", + group: EVALUATION, }), flag( "ground-truth", "expected answers for the sessions (JSON SessionMetadataShape[])", z.string().optional(), - { group: "Evaluation:", help: groundTruthHelp }, + { group: EVALUATION, help: groundTruthHelp }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index 32ca14728..b4b381c76 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -7,6 +7,11 @@ import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; +const RUNTIME_INVOCATION = "Runtime invocation:"; +const DATASET = "Dataset:"; +const CONFIGURATION = "Configuration:"; +const EVALUATION = "Evaluation:"; + const payloadTemplateHelp = `(JSON object) The request body sent to the Runtime for each dataset example. Every occurrence of {input} is replaced with that example's input, so the template describes the @@ -25,53 +30,53 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions", flags: [ flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), { - group: "Runtime invocation:", + group: RUNTIME_INVOCATION, }), flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { - group: "Runtime invocation:", + group: RUNTIME_INVOCATION, }), flag( "payload-template", "request body per example (JSON object); {input} is replaced with the input", z.string().optional(), - { group: "Runtime invocation:", help: payloadTemplateHelp }, + { group: RUNTIME_INVOCATION, help: payloadTemplateHelp }, ), flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), { - group: "Runtime invocation:", + group: RUNTIME_INVOCATION, sensitive: true, }), flag( "bearer-token", "CUSTOM_JWT bearer token (for JWT-auth Runtimes)", z.string().optional(), - { group: "Runtime invocation:", sensitive: true }, + { group: RUNTIME_INVOCATION, sensitive: true }, ), flag("user-id", "Runtime user ID", z.string().optional(), { - group: "Runtime invocation:", + group: RUNTIME_INVOCATION, }), flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional(), { - group: "Dataset:", + group: DATASET, }), flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), { - group: "Dataset:", + group: DATASET, }), flag( "ingestion-wait-ms", "ms to wait for span ingestion before grading (default 180000; 0 to skip)", z.coerce.number().int().nonnegative().optional(), - { group: "Dataset:" }, + { group: DATASET }, ), flag("name", "batch evaluation name (unique in the account)", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag("description", "description for the batch evaluation", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { - group: "Evaluation:", + group: EVALUATION, }), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx index b93adb1ef..92f8f51eb 100644 --- a/src/handlers/eval/batch-insights/run/index.tsx +++ b/src/handlers/eval/batch-insights/run/index.tsx @@ -7,6 +7,9 @@ import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +const CONFIGURATION = "Configuration:"; +const ANALYSIS = "Analysis:"; + const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis"; export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => @@ -15,23 +18,23 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => description: "start an asynchronous batch insights run over existing sessions", flags: [ flag("name", "batch insights name (must be unique in the account)", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag("description", "optional description", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), ...SessionSource.flags, flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT]), { - group: "Analysis:", + group: ANALYSIS, }), flag( "evaluators", "optional evaluator ID(s) to run alongside the insights", z.array(z.string()).optional(), - { group: "Analysis:" }, + { group: ANALYSIS }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index 1ebdb5ab2..ef22f4979 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -9,67 +9,72 @@ import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from ".. import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +const CONFIGURATION = "Configuration:"; +const SESSION_SOURCE = "Session source (choose exactly one):"; +const EVALUATION = "Evaluation:"; +const EXECUTION = "Execution:"; + export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "create", description: "create an online evaluation config", flags: [ flag("name", "the name of the online evaluation config", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }), flag( "description", "a description of the config's monitoring purpose", z.string().optional(), { - group: "Configuration:", + group: CONFIGURATION, }, ), flag( "enable-on-create", "whether to enable evaluation immediately (default true; pass false to create it paused)", z.enum(["true", "false"]).optional(), - { group: "Configuration:" }, + { group: CONFIGURATION }, ), flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { - group: "Session source (choose exactly one):", + group: SESSION_SOURCE, }), flag( "data-source-config", "the traces to sample (JSON DataSourceConfig), as an alternative to --agent", z.string().optional(), - { group: "Session source (choose exactly one):", help: onlineEvalDataSourceConfigHelp }, + { group: SESSION_SOURCE, help: onlineEvalDataSourceConfigHelp }, ), flag( "endpoint", "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", z.string().optional(), - { group: "Session source (choose exactly one):" }, + { group: SESSION_SOURCE }, ), flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional(), { - group: "Evaluation:", + group: EVALUATION, }), flag( "sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().min(0.01).max(100).optional(), - { group: "Evaluation:" }, + { group: EVALUATION }, ), flag( "session-timeout-minutes", "minutes of inactivity before a session is considered complete (1-1440, default 15)", z.number().int().min(1).max(1440).optional(), - { group: "Evaluation:" }, + { group: EVALUATION }, ), flag("filters", "trace filters (JSON Filter[])", z.string().optional(), { - group: "Evaluation:", + group: EVALUATION, help: filtersHelp, }), flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", z.string().optional(), - { group: "Execution:" }, + { group: EXECUTION }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index f5d8e895e..46fe3103f 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -10,68 +10,74 @@ import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from ".. import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +const TARGET = "Target:"; +const SESSION_SOURCE = "Session source:"; +const SOURCE_FILTERS = "Source filters:"; +const EVALUATION = "Evaluation:"; +const EXECUTION = "Execution:"; + export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "update", description: "update an online evaluation config", flags: [ flag("id", "the ID of the online evaluation config to update", z.string().optional(), { - group: "Target:", + group: TARGET, }), flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { - group: "Session source:", + group: SESSION_SOURCE, }), flag( "data-source-config", "replace the traces to sample (JSON DataSourceConfig)", z.string().optional(), - { group: "Session source:", help: onlineEvalDataSourceConfigHelp }, + { group: SESSION_SOURCE, help: onlineEvalDataSourceConfigHelp }, ), flag( "endpoint", "re-scope monitoring to a different agent endpoint qualifier", z.string().optional(), - { group: "Source filters:" }, + { group: SOURCE_FILTERS }, ), flag( "clear-endpoint", "reset the endpoint scope to the default qualifier (pass true)", z.enum(["true", "false"]).optional(), - { group: "Source filters:" }, + { group: SOURCE_FILTERS }, ), flag( "evaluators", "the ID(s) of the evaluators to apply (replaces the existing list)", z.array(z.string()).optional(), - { group: "Evaluation:" }, + { group: EVALUATION }, ), flag( "sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().min(0.01).max(100).optional(), - { group: "Evaluation:" }, + { group: EVALUATION }, ), flag( "session-timeout-minutes", "minutes of inactivity before a session is considered complete (1-1440)", z.number().int().min(1).max(1440).optional(), - { group: "Evaluation:" }, + { group: EVALUATION }, ), flag("filters", "replace the trace filters (JSON Filter[])", z.string().optional(), { - group: "Evaluation:", + group: EVALUATION, help: filtersHelp, }), flag( "role-arn", "replace the IAM role the online evaluation assumes", z.string().optional(), - { group: "Execution:" }, + { group: EXECUTION }, ), flag( "update-role", "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", z.enum(["true", "false"]).optional(), - { group: "Execution:" }, + { group: EXECUTION }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/sessionSource.tsx b/src/handlers/eval/sessionSource.tsx index b69c6d76a..40adc1f07 100644 --- a/src/handlers/eval/sessionSource.tsx +++ b/src/handlers/eval/sessionSource.tsx @@ -6,6 +6,9 @@ import { flag, type Flag } from "../../router"; import { assertMutuallyExclusiveFlags, parseJsonFlag } from "../utils"; import type { SessionSourceValue, SessionWindow } from "./types"; +const SESSION_SOURCE = "Session source (choose exactly one):"; +const SOURCE_FILTERS = "Source filters:"; + const dataSourceConfigHelp = `(JSON: tagged union object) Where sessions and traces are read from, for sources the --agent and --online-eval convenience flags cannot express. Only top-level key: @@ -37,38 +40,38 @@ Example: export class SessionSource { static readonly flags = [ flag("agent", "harness ID or Runtime ID whose sessions to use", z.string().optional(), { - group: "Session source (choose exactly one):", + group: SESSION_SOURCE, }), flag( "online-eval", "use sessions an online-eval config already sampled", z.string().optional(), - { group: "Session source (choose exactly one):" }, + { group: SESSION_SOURCE }, ), flag( "data-source-config", "the traces to read (JSON DataSourceConfig); escape hatch", z.string().optional(), - { group: "Session source (choose exactly one):", help: dataSourceConfigHelp }, + { group: SESSION_SOURCE, help: dataSourceConfigHelp }, ), flag( "endpoint", "Runtime endpoint qualifier (default DEFAULT; only with --agent)", z.string().optional(), - { group: "Source filters:" }, + { group: SOURCE_FILTERS }, ), flag("start-time", "window start (ISO-8601, with --end-time)", z.string().optional(), { - group: "Source filters:", + group: SOURCE_FILTERS, }), flag("end-time", "window end (ISO-8601, with --start-time)", z.string().optional(), { - group: "Source filters:", + group: SOURCE_FILTERS, }), flag( "session-ids", "specific session IDs (only with --agent)", z.array(z.string()).optional(), { - group: "Source filters:", + group: SOURCE_FILTERS, }, ), ] as const; From e580808345463431c8e39ff78179dbc178b2f3a8 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 11 Sep 2026 17:14:05 +0000 Subject: [PATCH 5/5] refactor(eval): inline help-group headings used only once A const earns its place when the string appears more than once; naming a single use only adds a hop. simulate's Evaluation, online-eval create's Execution, and update's Target headings are each used by one flag, so they go back to literals. The rest stay named. --- src/handlers/eval/batch-evaluation/simulate/index.tsx | 3 +-- src/handlers/eval/online-eval/create/index.tsx | 3 +-- src/handlers/eval/online-eval/update/index.tsx | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index b4b381c76..ff6c51ec7 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -10,7 +10,6 @@ import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; const RUNTIME_INVOCATION = "Runtime invocation:"; const DATASET = "Dataset:"; const CONFIGURATION = "Configuration:"; -const EVALUATION = "Evaluation:"; const payloadTemplateHelp = `(JSON object) The request body sent to the Runtime for each dataset example. Every occurrence @@ -76,7 +75,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => group: CONFIGURATION, }), flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { - group: EVALUATION, + group: "Evaluation:", }), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index ef22f4979..273b95c7b 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -12,7 +12,6 @@ import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; const CONFIGURATION = "Configuration:"; const SESSION_SOURCE = "Session source (choose exactly one):"; const EVALUATION = "Evaluation:"; -const EXECUTION = "Execution:"; export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ @@ -74,7 +73,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", z.string().optional(), - { group: EXECUTION }, + { group: "Execution:" }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 46fe3103f..6bdaa6807 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -10,7 +10,6 @@ import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from ".. import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; -const TARGET = "Target:"; const SESSION_SOURCE = "Session source:"; const SOURCE_FILTERS = "Source filters:"; const EVALUATION = "Evaluation:"; @@ -22,7 +21,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => description: "update an online evaluation config", flags: [ flag("id", "the ID of the online evaluation config to update", z.string().optional(), { - group: TARGET, + group: "Target:", }), flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { group: SESSION_SOURCE,