feat(eval): grouped help and command examples for the eval commands

src/handlers/eval/batch-evaluation/evaluate/index.tsx

@@ -8,21 +8,57 @@ import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore";
 import { coreOptsFromCtx, parseJsonFlag } from "../../../utils";
 import { SessionSource } from "../../sessionSource";
 
+const CONFIGURATION = "Configuration:";
+const EVALUATION = "Evaluation:";
+
+const groundTruthHelp = `(JSON: list of objects)
+Expected answers for the sessions being evaluated, so an evaluator can score a
+response against a reference instead of judging it on its own. Each entry names
+one session; omit an entry for a session that has no reference answer.
+
+Accepts inline JSON, file://<path>, or - to read stdin.
+
+JSON syntax:
+  [
+    {
+      "sessionId": "string",         // [required] the session the reference applies to
+      "testScenarioId": "string",    // groups sessions replaying the same scenario
+      "groundTruth": {
+        "inline": "string"           // the expected answer
+      }
+    },
+    ...
+  ]
+
+Example:
+  --ground-truth '[{"sessionId":"session-123","groundTruth":{"inline":"The order shipped on Tuesday."}}]'
+
+  --ground-truth file://ground-truth.json`;
+
 export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
   createHandler({
     name: "evaluate",
     description: "evaluate existing sessions service-side (async; returns a job ID)",
     flags: [
+      flag("name", "batch evaluation name (must be unique in the account)", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("description", "optional description", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
       ...SessionSource.flags,
-      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), {
+        group: EVALUATION,
+      }),
       flag(
         "ground-truth",
-        "session ground truth (JSON SessionMetadataShape[]; inline, file://<path>, or -)",
+        "expected answers for the sessions (JSON SessionMetadataShape[])",
         z.string().optional(),
+        { group: EVALUATION, help: groundTruthHelp },
       ),
-      flag("name", "batch evaluation name (must be unique in the account)", z.string().optional()),
-      flag("description", "optional description", z.string().optional()),
-      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()),
     ],
     handle: async (ctx, flags) => {
       if (!flags["name"]) {

src/handlers/eval/batch-evaluation/simulate/index.tsx

@@ -7,41 +7,77 @@ import type { Core } from "../../../types";
 import { coreOptsFromCtx } from "../../../utils";
 import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request";
 
+const RUNTIME_INVOCATION = "Runtime invocation:";
+const DATASET = "Dataset:";
+const CONFIGURATION = "Configuration:";
+const EVALUATION = "Evaluation:";
+
+const payloadTemplateHelp = `(JSON object)
+The request body sent to the Runtime for each dataset example. Every occurrence
+of {input} is replaced with that example's input, so the template describes the
+shape your agent expects and {input} marks where the prompt goes.
+
+Example:
+  --payload-template '{"prompt":"{input}"}'
+
+  --payload-template '{"messages":[{"role":"user","content":"{input}"}],"stream":false}'`;
+
 // Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror
 // `runtime invoke`.
 export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) =>
   createHandler({
     name: "simulate",
     description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions",
     flags: [
-      flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional()),
-      flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()),
+      flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), {
+        group: RUNTIME_INVOCATION,
+      }),
+      flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), {
+        group: RUNTIME_INVOCATION,
+      }),
       flag(
         "payload-template",
-        'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}',
+        "request body per example (JSON object); {input} is replaced with the input",
         z.string().optional(),
+        { group: RUNTIME_INVOCATION, help: payloadTemplateHelp },
       ),
       flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), {
+        group: RUNTIME_INVOCATION,
         sensitive: true,
       }),
       flag(
         "bearer-token",
         "CUSTOM_JWT bearer token (for JWT-auth Runtimes)",
         z.string().optional(),
-        { sensitive: true },
+        { group: RUNTIME_INVOCATION, sensitive: true },
       ),
-      flag("user-id", "Runtime user ID", z.string().optional()),
-      flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()),
-      flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()),
-      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
-      flag("name", "batch evaluation name (unique in the account)", z.string().optional()),
-      flag("description", "description for the batch evaluation", z.string().optional()),
-      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()),
+      flag("user-id", "Runtime user ID", z.string().optional(), {
+        group: RUNTIME_INVOCATION,
+      }),
+      flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional(), {
+        group: DATASET,
+      }),
+      flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), {
+        group: DATASET,
+      }),
       flag(
         "ingestion-wait-ms",
         "ms to wait for span ingestion before grading (default 180000; 0 to skip)",
         z.coerce.number().int().nonnegative().optional(),
+        { group: DATASET },
       ),
+      flag("name", "batch evaluation name (unique in the account)", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("description", "description for the batch evaluation", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), {
+        group: EVALUATION,
+      }),
     ],
     handle: async (ctx, flags) => {
       if (!flags["runtime-id"])

src/handlers/eval/batch-insights/run/index.tsx

@@ -7,23 +7,35 @@ import type { Core } from "../../../types";
 import { coreOptsFromCtx } from "../../../utils";
 import { SessionSource } from "../../sessionSource";
 
+const CONFIGURATION = "Configuration:";
+const ANALYSIS = "Analysis:";
+
 const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis";
 
 export const createRunBatchInsightsHandler = (core: Core, io: AppIO) =>
   createHandler({
     name: "run",
     description: "start an asynchronous batch insights run over existing sessions",
     flags: [
+      flag("name", "batch insights name (must be unique in the account)", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("description", "optional description", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
       ...SessionSource.flags,
-      flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT])),
+      flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT]), {
+        group: ANALYSIS,
+      }),
       flag(
         "evaluators",
         "optional evaluator ID(s) to run alongside the insights",
         z.array(z.string()).optional(),
+        { group: ANALYSIS },
       ),
-      flag("name", "batch insights name (must be unique in the account)", z.string().optional()),
-      flag("description", "optional description", z.string().optional()),
-      flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional()),
     ],
     handle: async (ctx, flags) => {
       if (!flags["name"]) {

src/handlers/eval/online-eval/create/index.tsx

@@ -6,54 +6,75 @@ import { JsonRendererKey } from "../../../../tui";
 import { SourceResolver, type AppIO } from "../../../../io";
 import type { Core } from "../../../types";
 import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils";
+import { filtersHelp } from "../filtersHelp";
+import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp";
+
+const CONFIGURATION = "Configuration:";
+const SESSION_SOURCE = "Session source (choose exactly one):";
+const EVALUATION = "Evaluation:";
+const EXECUTION = "Execution:";
 
 export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) =>
   createHandler({
     name: "create",
     description: "create an online evaluation config",
     flags: [
-      flag("name", "the name of the online evaluation config", z.string().optional()),
-      flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional()),
+      flag("name", "the name of the online evaluation config", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
       flag(
-        "endpoint",
-        "the agent endpoint qualifier to scope monitoring to (default DEFAULT)",
+        "description",
+        "a description of the config's monitoring purpose",
         z.string().optional(),
+        {
+          group: CONFIGURATION,
+        },
+      ),
+      flag(
+        "enable-on-create",
+        "whether to enable evaluation immediately (default true; pass false to create it paused)",
+        z.enum(["true", "false"]).optional(),
+        { group: CONFIGURATION },
       ),
+      flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), {
+        group: SESSION_SOURCE,
+      }),
       flag(
         "data-source-config",
-        "the traces to evaluate (JSON DataSourceConfig; inline, file://<path>, or - for stdin), as an alternative to --agent",
+        "the traces to sample (JSON DataSourceConfig), as an alternative to --agent",
+        z.string().optional(),
+        { group: SESSION_SOURCE, help: onlineEvalDataSourceConfigHelp },
+      ),
+      flag(
+        "endpoint",
+        "the agent endpoint qualifier to scope monitoring to (default DEFAULT)",
         z.string().optional(),
+        { group: SESSION_SOURCE },
       ),
-      flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()),
+      flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional(), {
+        group: EVALUATION,
+      }),
       flag(
         "sampling-rate",
         "percentage of sessions to sample (0.01-100)",
         z.number().min(0.01).max(100).optional(),
+        { group: EVALUATION },
       ),
       flag(
         "session-timeout-minutes",
         "minutes of inactivity before a session is considered complete (1-1440, default 15)",
         z.number().int().min(1).max(1440).optional(),
+        { group: EVALUATION },
       ),
-      flag(
-        "filters",
-        "trace filters (JSON Filter[]; inline, file://<path>, or - for stdin)",
-        z.string().optional(),
-      ),
+      flag("filters", "trace filters (JSON Filter[])", z.string().optional(), {
+        group: EVALUATION,
+        help: filtersHelp,
+      }),
       flag(
         "role-arn",
         "IAM role the online evaluation assumes (default auto-provisioned)",
         z.string().optional(),
-      ),
-      flag(
-        "enable-on-create",
-        "whether to enable evaluation immediately (default true; pass false to create it paused)",
-        z.enum(["true", "false"]).optional(),
-      ),
-      flag(
-        "description",
-        "a description of the config's monitoring purpose",
-        z.string().optional(),
+        { group: EXECUTION },
       ),
     ],
     handle: async (ctx, flags) => {

src/handlers/eval/online-eval/dataSourceConfigHelp.tsx

@@ -0,0 +1,20 @@
+export const onlineEvalDataSourceConfigHelp = `(JSON: tagged union object)
+Which traces are sampled, for sources the --agent convenience flag cannot
+express. Only top-level key: cloudWatchLogs.
+
+Accepts inline JSON, file://<path>, or - to read stdin.
+
+JSON syntax:
+  {
+    "cloudWatchLogs": {
+      "serviceNames": ["string", ...],          // [required] e.g. "my_agent.DEFAULT"
+      "logGroupNames": ["string", ...],         // exact group names
+      "logGroupNamePrefixes": ["string", ...]   // or match by prefix instead
+    }                                           // supply one of the two name lists
+  }
+
+API reference:
+  https://docs.aws.amazon.com/bedrock-agentcore-control/latest/APIReference/API_DataSourceConfig.html
+
+Example:
+  --data-source-config '{"cloudWatchLogs":{"logGroupNames":["/aws/bedrock-agentcore/runtimes/my-runtime-DEFAULT"],"serviceNames":["my_agent.DEFAULT"]}}'`;

src/handlers/eval/online-eval/filtersHelp.tsx

@@ -0,0 +1,26 @@
+export const filtersHelp = `(JSON: list of objects)
+Narrows which sampled traces are evaluated. A trace must match every filter in
+the list. Omit it to evaluate every sampled trace.
+
+Accepts inline JSON, file://<path>, or - to read stdin.
+
+JSON syntax:
+  [
+    {
+      "key": "string",       // [required] trace field to filter on
+      "operator": "Equals" | "NotEquals" | "Contains" | "NotContains"
+                | "GreaterThan" | "GreaterThanOrEqual"
+                | "LessThan" | "LessThanOrEqual",   // [required]
+      "value": {                                    // [required] exactly one key
+        "stringValue": "string",
+        "doubleValue": number,
+        "booleanValue": true | false
+      }
+    },
+    ...
+  ]
+
+Example:
+  --filters '[{"key":"attributes.customer_tier","operator":"Equals","value":{"stringValue":"enterprise"}}]'
+
+  --filters file://filters.json`;