diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index f6c73c512..8b95678ad 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -8,21 +8,57 @@ import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +const CONFIGURATION = "Configuration:"; +const EVALUATION = "Evaluation:"; + +const groundTruthHelp = `(JSON: list of objects) +Expected answers for the sessions being evaluated, so an evaluator can score a +response against a reference instead of judging it on its own. Each entry names +one session; omit an entry for a session that has no reference answer. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + [ + { + "sessionId": "string", // [required] the session the reference applies to + "testScenarioId": "string", // groups sessions replaying the same scenario + "groundTruth": { + "inline": "string" // the expected answer + } + }, + ... + ] + +Example: + --ground-truth '[{"sessionId":"session-123","groundTruth":{"inline":"The order shipped on Tuesday."}}]' + + --ground-truth file://ground-truth.json`; + export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ name: "evaluate", description: "evaluate existing sessions service-side (async; returns a job ID)", flags: [ + flag("name", "batch evaluation name (must be unique in the account)", z.string().optional(), { + group: CONFIGURATION, + }), + flag("description", "optional description", z.string().optional(), { + group: CONFIGURATION, + }), + flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { + group: CONFIGURATION, + }), ...SessionSource.flags, - flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { + group: EVALUATION, + }), flag( "ground-truth", - "session ground truth (JSON SessionMetadataShape[]; inline, file://, or -)", + "expected answers for the sessions (JSON SessionMetadataShape[])", z.string().optional(), + { group: EVALUATION, help: groundTruthHelp }, ), - flag("name", "batch evaluation name (must be unique in the account)", z.string().optional()), - flag("description", "optional description", z.string().optional()), - flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), ], handle: async (ctx, flags) => { if (!flags["name"]) { diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index a880a3ddf..ff6c51ec7 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -7,6 +7,20 @@ import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; +const RUNTIME_INVOCATION = "Runtime invocation:"; +const DATASET = "Dataset:"; +const CONFIGURATION = "Configuration:"; + +const payloadTemplateHelp = `(JSON object) +The request body sent to the Runtime for each dataset example. Every occurrence +of {input} is replaced with that example's input, so the template describes the +shape your agent expects and {input} marks where the prompt goes. + +Example: + --payload-template '{"prompt":"{input}"}' + + --payload-template '{"messages":[{"role":"user","content":"{input}"}],"stream":false}'`; + // Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror // `runtime invoke`. export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => @@ -14,34 +28,55 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => name: "simulate", description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions", flags: [ - flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional()), - flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), { + group: RUNTIME_INVOCATION, + }), + flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { + group: RUNTIME_INVOCATION, + }), flag( "payload-template", - 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + "request body per example (JSON object); {input} is replaced with the input", z.string().optional(), + { group: RUNTIME_INVOCATION, help: payloadTemplateHelp }, ), flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), { + group: RUNTIME_INVOCATION, sensitive: true, }), flag( "bearer-token", "CUSTOM_JWT bearer token (for JWT-auth Runtimes)", z.string().optional(), - { sensitive: true }, + { group: RUNTIME_INVOCATION, sensitive: true }, ), - flag("user-id", "Runtime user ID", z.string().optional()), - flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()), - flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()), - flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), - flag("name", "batch evaluation name (unique in the account)", z.string().optional()), - flag("description", "description for the batch evaluation", z.string().optional()), - flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), + flag("user-id", "Runtime user ID", z.string().optional(), { + group: RUNTIME_INVOCATION, + }), + flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional(), { + group: DATASET, + }), + flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), { + group: DATASET, + }), flag( "ingestion-wait-ms", "ms to wait for span ingestion before grading (default 180000; 0 to skip)", z.coerce.number().int().nonnegative().optional(), + { group: DATASET }, ), + flag("name", "batch evaluation name (unique in the account)", z.string().optional(), { + group: CONFIGURATION, + }), + flag("description", "description for the batch evaluation", z.string().optional(), { + group: CONFIGURATION, + }), + flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), { + group: CONFIGURATION, + }), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { + group: "Evaluation:", + }), ], handle: async (ctx, flags) => { if (!flags["runtime-id"]) diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx index 6d9d2a037..92f8f51eb 100644 --- a/src/handlers/eval/batch-insights/run/index.tsx +++ b/src/handlers/eval/batch-insights/run/index.tsx @@ -7,6 +7,9 @@ import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +const CONFIGURATION = "Configuration:"; +const ANALYSIS = "Analysis:"; + const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis"; export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => @@ -14,16 +17,25 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => name: "run", description: "start an asynchronous batch insights run over existing sessions", flags: [ + flag("name", "batch insights name (must be unique in the account)", z.string().optional(), { + group: CONFIGURATION, + }), + flag("description", "optional description", z.string().optional(), { + group: CONFIGURATION, + }), + flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional(), { + group: CONFIGURATION, + }), ...SessionSource.flags, - flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT])), + flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT]), { + group: ANALYSIS, + }), flag( "evaluators", "optional evaluator ID(s) to run alongside the insights", z.array(z.string()).optional(), + { group: ANALYSIS }, ), - flag("name", "batch insights name (must be unique in the account)", z.string().optional()), - flag("description", "optional description", z.string().optional()), - flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional()), ], handle: async (ctx, flags) => { if (!flags["name"]) { diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index d825e1001..273b95c7b 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -6,54 +6,74 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { filtersHelp } from "../filtersHelp"; +import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; + +const CONFIGURATION = "Configuration:"; +const SESSION_SOURCE = "Session source (choose exactly one):"; +const EVALUATION = "Evaluation:"; export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "create", description: "create an online evaluation config", flags: [ - flag("name", "the name of the online evaluation config", z.string().optional()), - flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional()), + flag("name", "the name of the online evaluation config", z.string().optional(), { + group: CONFIGURATION, + }), flag( - "endpoint", - "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", + "description", + "a description of the config's monitoring purpose", z.string().optional(), + { + group: CONFIGURATION, + }, + ), + flag( + "enable-on-create", + "whether to enable evaluation immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + { group: CONFIGURATION }, ), + flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { + group: SESSION_SOURCE, + }), flag( "data-source-config", - "the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin), as an alternative to --agent", + "the traces to sample (JSON DataSourceConfig), as an alternative to --agent", + z.string().optional(), + { group: SESSION_SOURCE, help: onlineEvalDataSourceConfigHelp }, + ), + flag( + "endpoint", + "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", z.string().optional(), + { group: SESSION_SOURCE }, ), - flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()), + flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional(), { + group: EVALUATION, + }), flag( "sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().min(0.01).max(100).optional(), + { group: EVALUATION }, ), flag( "session-timeout-minutes", "minutes of inactivity before a session is considered complete (1-1440, default 15)", z.number().int().min(1).max(1440).optional(), + { group: EVALUATION }, ), - flag( - "filters", - "trace filters (JSON Filter[]; inline, file://, or - for stdin)", - z.string().optional(), - ), + flag("filters", "trace filters (JSON Filter[])", z.string().optional(), { + group: EVALUATION, + help: filtersHelp, + }), flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", z.string().optional(), - ), - flag( - "enable-on-create", - "whether to enable evaluation immediately (default true; pass false to create it paused)", - z.enum(["true", "false"]).optional(), - ), - flag( - "description", - "a description of the config's monitoring purpose", - z.string().optional(), + { group: "Execution:" }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/online-eval/dataSourceConfigHelp.tsx b/src/handlers/eval/online-eval/dataSourceConfigHelp.tsx new file mode 100644 index 000000000..e079181fc --- /dev/null +++ b/src/handlers/eval/online-eval/dataSourceConfigHelp.tsx @@ -0,0 +1,20 @@ +export const onlineEvalDataSourceConfigHelp = `(JSON: tagged union object) +Which traces are sampled, for sources the --agent convenience flag cannot +express. Only top-level key: cloudWatchLogs. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchLogs": { + "serviceNames": ["string", ...], // [required] e.g. "my_agent.DEFAULT" + "logGroupNames": ["string", ...], // exact group names + "logGroupNamePrefixes": ["string", ...] // or match by prefix instead + } // supply one of the two name lists + } + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore-control/latest/APIReference/API_DataSourceConfig.html + +Example: + --data-source-config '{"cloudWatchLogs":{"logGroupNames":["/aws/bedrock-agentcore/runtimes/my-runtime-DEFAULT"],"serviceNames":["my_agent.DEFAULT"]}}'`; diff --git a/src/handlers/eval/online-eval/filtersHelp.tsx b/src/handlers/eval/online-eval/filtersHelp.tsx new file mode 100644 index 000000000..35e67aa87 --- /dev/null +++ b/src/handlers/eval/online-eval/filtersHelp.tsx @@ -0,0 +1,26 @@ +export const filtersHelp = `(JSON: list of objects) +Narrows which sampled traces are evaluated. A trace must match every filter in +the list. Omit it to evaluate every sampled trace. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + [ + { + "key": "string", // [required] trace field to filter on + "operator": "Equals" | "NotEquals" | "Contains" | "NotContains" + | "GreaterThan" | "GreaterThanOrEqual" + | "LessThan" | "LessThanOrEqual", // [required] + "value": { // [required] exactly one key + "stringValue": "string", + "doubleValue": number, + "booleanValue": true | false + } + }, + ... + ] + +Example: + --filters '[{"key":"attributes.customer_tier","operator":"Equals","value":{"stringValue":"enterprise"}}]' + + --filters file://filters.json`; diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 664f78470..6bdaa6807 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -7,54 +7,76 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { filtersHelp } from "../filtersHelp"; +import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; + +const SESSION_SOURCE = "Session source:"; +const SOURCE_FILTERS = "Source filters:"; +const EVALUATION = "Evaluation:"; +const EXECUTION = "Execution:"; export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "update", description: "update an online evaluation config", flags: [ - flag("id", "the ID of the online evaluation config to update", z.string().optional()), + flag("id", "the ID of the online evaluation config to update", z.string().optional(), { + group: "Target:", + }), + flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { + group: SESSION_SOURCE, + }), flag( - "sampling-rate", - "percentage of sessions to sample (0.01-100)", - z.number().min(0.01).max(100).optional(), + "data-source-config", + "replace the traces to sample (JSON DataSourceConfig)", + z.string().optional(), + { group: SESSION_SOURCE, help: onlineEvalDataSourceConfigHelp }, ), flag( - "session-timeout-minutes", - "minutes of inactivity before a session is considered complete (1-1440)", - z.number().int().min(1).max(1440).optional(), + "endpoint", + "re-scope monitoring to a different agent endpoint qualifier", + z.string().optional(), + { group: SOURCE_FILTERS }, ), flag( - "filters", - "trace filters (JSON Filter[]; inline, file://, or - for stdin)", - z.string().optional(), + "clear-endpoint", + "reset the endpoint scope to the default qualifier (pass true)", + z.enum(["true", "false"]).optional(), + { group: SOURCE_FILTERS }, ), flag( "evaluators", "the ID(s) of the evaluators to apply (replaces the existing list)", z.array(z.string()).optional(), + { group: EVALUATION }, ), - flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional()), flag( - "endpoint", - "re-scope monitoring to a different agent endpoint qualifier", - z.string().optional(), + "sampling-rate", + "percentage of sessions to sample (0.01-100)", + z.number().min(0.01).max(100).optional(), + { group: EVALUATION }, ), flag( - "clear-endpoint", - "reset the endpoint scope to the default qualifier (pass true)", - z.enum(["true", "false"]).optional(), + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete (1-1440)", + z.number().int().min(1).max(1440).optional(), + { group: EVALUATION }, ), + flag("filters", "replace the trace filters (JSON Filter[])", z.string().optional(), { + group: EVALUATION, + help: filtersHelp, + }), flag( - "data-source-config", - "replace the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin)", + "role-arn", + "replace the IAM role the online evaluation assumes", z.string().optional(), + { group: EXECUTION }, ), - flag("role-arn", "replace the IAM role the online evaluation assumes", z.string().optional()), flag( "update-role", "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", z.enum(["true", "false"]).optional(), + { group: EXECUTION }, ), ], handle: async (ctx, flags) => { diff --git a/src/handlers/eval/sessionSource.tsx b/src/handlers/eval/sessionSource.tsx index aa8facd96..40adc1f07 100644 --- a/src/handlers/eval/sessionSource.tsx +++ b/src/handlers/eval/sessionSource.tsx @@ -6,38 +6,73 @@ import { flag, type Flag } from "../../router"; import { assertMutuallyExclusiveFlags, parseJsonFlag } from "../utils"; import type { SessionSourceValue, SessionWindow } from "./types"; +const SESSION_SOURCE = "Session source (choose exactly one):"; +const SOURCE_FILTERS = "Source filters:"; + +const dataSourceConfigHelp = `(JSON: tagged union object) +Where sessions and traces are read from, for sources the --agent and +--online-eval convenience flags cannot express. Only top-level key: +cloudWatchLogs. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchLogs": { + "logGroupNames": ["string", ...], // [required] groups holding the traces + "serviceNames": ["string", ...], // e.g. "my_agent.DEFAULT" + "filterConfig": { + "sessionIds": ["string", ...], + "sessionFilterConfig": { + "startTime": "timestamp", + "endTime": "timestamp" + } + } + } + } + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_DataSourceConfig.html + +Example: + --data-source-config '{"cloudWatchLogs":{"logGroupNames":["/aws/bedrock-agentcore/runtimes/support_agent-AbC123XyZ9-DEFAULT"],"serviceNames":["support_agent.DEFAULT"],"filterConfig":{"sessionIds":["session-123"]}}}'`; + export class SessionSource { static readonly flags = [ - flag("agent", "source: harness ID or Runtime ID whose sessions to use", z.string().optional()), - flag( - "endpoint", - "Runtime endpoint qualifier (default DEFAULT; only with --agent)", - z.string().optional(), - ), + flag("agent", "harness ID or Runtime ID whose sessions to use", z.string().optional(), { + group: SESSION_SOURCE, + }), flag( "online-eval", - "source: use sessions an online-eval config already sampled", + "use sessions an online-eval config already sampled", z.string().optional(), + { group: SESSION_SOURCE }, ), flag( "data-source-config", - "source: raw DataSourceConfig JSON (inline, file://, or -); escape hatch", + "the traces to read (JSON DataSourceConfig); escape hatch", z.string().optional(), + { group: SESSION_SOURCE, help: dataSourceConfigHelp }, ), flag( - "start-time", - "time filter: window start (ISO-8601, with --end-time)", - z.string().optional(), - ), - flag( - "end-time", - "time filter: window end (ISO-8601, with --start-time)", + "endpoint", + "Runtime endpoint qualifier (default DEFAULT; only with --agent)", z.string().optional(), + { group: SOURCE_FILTERS }, ), + flag("start-time", "window start (ISO-8601, with --end-time)", z.string().optional(), { + group: SOURCE_FILTERS, + }), + flag("end-time", "window end (ISO-8601, with --start-time)", z.string().optional(), { + group: SOURCE_FILTERS, + }), flag( "session-ids", - "filter: specific session IDs (only with --agent)", + "specific session IDs (only with --agent)", z.array(z.string()).optional(), + { + group: SOURCE_FILTERS, + }, ), ] as const; diff --git a/src/router/flags.tsx b/src/router/flags.tsx index 6314cf223..1f59770a1 100644 --- a/src/router/flags.tsx +++ b/src/router/flags.tsx @@ -32,6 +32,9 @@ export function toOption(flag: Flag): Option { if (info.required && !info.boolean) { option.makeOptionMandatory(true); } + if (flag.group) { + option.helpGroup(flag.group); + } return option; } diff --git a/src/router/handler.tsx b/src/router/handler.tsx index 146c6dff2..abb24834b 100644 --- a/src/router/handler.tsx +++ b/src/router/handler.tsx @@ -14,6 +14,7 @@ export interface Flag { // Its first line is the type annotation shown next to the flag name; the // remaining lines are the body — prose, JSON syntax, examples. help?: string; + group?: string; // sensitive flags are redacted from debug logs by the withLogging middleware. sensitive?: boolean; } @@ -34,9 +35,16 @@ export function flag( name: N, description: string, schema: z.ZodType, - options?: { help?: string; sensitive?: boolean }, + options?: { help?: string; group?: string; sensitive?: boolean }, ): Flag { - return { name, description, schema, help: options?.help, sensitive: options?.sensitive }; + return { + name, + description, + schema, + help: options?.help, + group: options?.group, + sensitive: options?.sensitive, + }; } // globalFlag constructs a GlobalFlag. The returned value doubles as the typed diff --git a/src/router/router.test.ts b/src/router/router.test.ts index ecdc19726..9808ed240 100644 --- a/src/router/router.test.ts +++ b/src/router/router.test.ts @@ -899,3 +899,61 @@ test("--version is an unknown option on a router without a version", async () => code: "commander.unknownOption", }); }); + +test("grouped flags render under their headings, ungrouped ones under Options", async () => { + const evaluate = createHandler({ + name: "evaluate", + description: "", + flags: [ + flag("agent", "the agent", z.string().optional(), { group: "Session source:" }), + flag("start-time", "window start", z.string().optional(), { group: "Source filters:" }), + flag("end-time", "window end", z.string().optional(), { group: "Source filters:" }), + flag("name", "the name", z.string().optional()), + ], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(evaluate); + + const out = await helpOutput(root, ["app", "evaluate", "--help"]); + + expect(out).toContain("Session source:"); + expect(out).toContain("Source filters:"); + expect(out.indexOf("Session source:")).toBeLessThan(out.indexOf("Source filters:")); + expect(out.indexOf("--agent")).toBeGreaterThan(out.indexOf("Session source:")); + expect(out.indexOf("--start-time")).toBeGreaterThan(out.indexOf("Source filters:")); + expect(out.indexOf("--end-time")).toBeGreaterThan(out.indexOf("Source filters:")); + expect(out.indexOf("--name")).toBeGreaterThan(out.indexOf("Options:")); +}); + +test("a command that groups its flags moves the generated help into Other options", async () => { + const grouped = createHandler({ + name: "grouped", + description: "", + flags: [flag("agent", "the agent", z.string().optional(), { group: "Session source:" })], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(grouped); + + const out = await helpOutput(root, ["app", "grouped", "--help"]); + + expect(out).toContain("Other options:"); + expect(out.indexOf("-h, --help")).toBeGreaterThan(out.indexOf("Other options:")); +}); + +test("a command without grouped flags leaves the generated help in Options", async () => { + const plain = createHandler({ + name: "plain", + description: "", + flags: [flag("id", "the id", z.string().optional())], + handle: async () => {}, + }); + const root = new Router("app"); + root.handler(plain); + + const out = await helpOutput(root, ["app", "plain", "--help"]); + + expect(out).not.toContain("Other options:"); + expect(out).toContain("-h, --help"); +}); diff --git a/src/router/router.tsx b/src/router/router.tsx index 7fdbf1da5..df938de28 100644 --- a/src/router/router.tsx +++ b/src/router/router.tsx @@ -4,7 +4,7 @@ import { type Context, type ContextKey, ValueContext, contextKey } from "./conte import { applyGlobalFlags, formatParameterDetails, parseFlags, toOption } from "./flags"; import { parseArguments, toCommanderArgument } from "./args"; -import { Command, CommanderError } from "commander"; +import { Command, CommanderError, Option } from "commander"; import { InputValidationError } from "../errors"; import type { Logger } from "../logging"; import type { GlobalConfigAccessor } from "../globalConfig"; @@ -187,6 +187,12 @@ export function compile( declareFlags(c, ownFlags); declareArguments(c, node.arguments()); + if (ownFlags.some((f) => f.group)) { + c.addHelpOption( + new Option("-h, --help", "display help for command").helpGroup("Other options:"), + ); + } + // Flags with long-form documentation get a "Parameter details" section after // the option list in `--help` output. const parameterDetails = formatParameterDetails(ownFlags);