From 289eeaebba0c07cd91cc298083bb2141b385d58c Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 9 Sep 2026 22:16:30 +0000 Subject: [PATCH] feat(eval): rename --evaluator to --evaluators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flag has always been variadic — every one of these commands accepts a list of evaluator IDs — but the singular name read as though it took one. Renamed across all eight commands that take the list, so the CLI does not end up with two spellings for the same concept. No deprecated alias: the refactor CLI has not shipped GA, so there are no scripts to keep working. Left alone deliberately: `project remove --evaluator`, where the value names a resource *type* rather than a list, and the `eval` router's TUI subcommand names, which are command names rather than flags. --- .../eval/ab-test/ab-test.fixture.test.tsx | 6 ++-- .../batch-evaluation.fixture.test.tsx | 4 +-- .../batch-evaluation.test.tsx | 21 +++++++++---- .../eval/batch-evaluation/evaluate/index.tsx | 8 ++--- .../eval/batch-evaluation/simulate/index.tsx | 8 ++--- .../batch-insights/batch-insights.test.tsx | 2 +- .../eval/batch-insights/run/index.tsx | 4 +-- src/handlers/eval/ondemand/evaluate/index.tsx | 8 ++--- .../eval/ondemand/ondemand.fixture.test.tsx | 4 +-- src/handlers/eval/ondemand/ondemand.test.tsx | 28 ++++++++++------- src/handlers/eval/ondemand/simulate/index.tsx | 8 ++--- .../eval/online-eval/create/index.tsx | 8 ++--- .../eval/online-eval/online-eval.test.tsx | 16 +++++----- .../eval/online-eval/update/index.tsx | 4 +-- .../project/add/online-eval/index.test.ts | 30 +++++++++---------- src/handlers/project/add/online-eval/index.ts | 4 +-- src/handlers/project/remove/index.test.ts | 2 +- 17 files changed, 91 insertions(+), 74 deletions(-) diff --git a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx index b5429f5d9..0812a5c95 100644 --- a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx @@ -199,7 +199,7 @@ describe("eval ab-test config-based run", () => { ONLINE_EVAL_NAME, "--agent", AGENT_ID, - "--evaluator", + "--evaluators", EVALUATOR_ID, "--sampling-rate", "100", @@ -280,7 +280,7 @@ describe("eval ab-test target-based run", () => { TB_ONLINE_EVAL_C, "--agent", AGENT_ID, - "--evaluator", + "--evaluators", EVALUATOR_ID, "--sampling-rate", "100", @@ -300,7 +300,7 @@ describe("eval ab-test target-based run", () => { TB_ONLINE_EVAL_T1, "--agent", AGENT_ID, - "--evaluator", + "--evaluators", EVALUATOR_ID, "--sampling-rate", "100", diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx index b3b8a8f62..a690811e3 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx @@ -128,7 +128,7 @@ describe("eval batch-evaluation (fixture-backed)", () => { "evaluate", "--agent", FIXTURE_EVAL_AGENT, - "--evaluator", + "--evaluators", "Builtin.Helpfulness", "--name", FIXTURE_EVAL_NAME, @@ -172,7 +172,7 @@ describe("eval batch-evaluation (fixture-backed)", () => { '{"prompt":"{input}"}', "--dataset", FIXTURE_SIMULATE_DATASET, - "--evaluator", + "--evaluators", "Builtin.Helpfulness", "--name", FIXTURE_SIMULATE_NAME, diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index e76da4395..98d49f7b2 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -172,7 +172,7 @@ describe("eval batch-evaluation simulate", () => { '{"prompt":"{input}"}', "--dataset", "/tmp/ds.jsonl", - "--evaluator", + "--evaluators", "Builtin.Helpfulness", "--name", "sim-1", @@ -181,18 +181,27 @@ describe("eval batch-evaluation simulate", () => { test.each<[RegExp, string[]]>([ [ /--runtime-id/, - ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], + [ + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--evaluators", + "E", + "--name", + "n", + ], ], [ /--payload-template/, - ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluators", "E", "--name", "n"], ], [ /--dataset/, - ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E", "--name", "n"], + ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluators", "E", "--name", "n"], ], [ - /--evaluator/, + /--evaluators/, [ "--runtime-id", "r-1", @@ -213,7 +222,7 @@ describe("eval batch-evaluation simulate", () => { "{}", "--dataset", "/tmp/ds.jsonl", - "--evaluator", + "--evaluators", "E", ], ], diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index 3d4565380..f6c73c512 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -14,7 +14,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => description: "evaluate existing sessions service-side (async; returns a job ID)", flags: [ ...SessionSource.flags, - flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), flag( "ground-truth", "session ground truth (JSON SessionMetadataShape[]; inline, file://, or -)", @@ -28,9 +28,9 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => if (!flags["name"]) { throw new InputValidationError("required option '--name ' not specified"); } - if (!flags["evaluator"] || flags["evaluator"].length === 0) { + if (!flags["evaluators"] || flags["evaluators"].length === 0) { throw new InputValidationError( - "required option '--evaluator ' not specified", + "required option '--evaluators ' not specified", ); } @@ -46,7 +46,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => { name: flags["name"], description: flags["description"], - evaluatorIds: flags["evaluator"], + evaluatorIds: flags["evaluators"], source, groundTruth, kmsKeyArn: flags["kms-key-arn"], diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index 45c18641d..a880a3ddf 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -33,7 +33,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => flag("user-id", "Runtime user ID", z.string().optional()), flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()), flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()), - flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), flag("name", "batch evaluation name (unique in the account)", z.string().optional()), flag("description", "description for the batch evaluation", z.string().optional()), flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), @@ -51,9 +51,9 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => } if (!flags["dataset"]) throw new InputValidationError("required option '--dataset' not specified"); - if (!flags["evaluator"]?.length) { + if (!flags["evaluators"]?.length) { throw new InputValidationError( - "required option '--evaluator ' not specified", + "required option '--evaluators ' not specified", ); } if (!flags["name"]) @@ -95,7 +95,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => { name: flags["name"], description: flags["description"], - evaluatorIds: flags["evaluator"], + evaluatorIds: flags["evaluators"], source: { origin: "agent", agent: flags["runtime-id"], diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx index 5a0837ad1..620ec9720 100644 --- a/src/handlers/eval/batch-insights/batch-insights.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.test.tsx @@ -137,7 +137,7 @@ describe("eval batch-insights run", () => { "--insight", "Builtin.Insight.UserIntent", "Builtin.Insight.ExecutionSummary", - "--evaluator", + "--evaluators", "Builtin.Helpfulness", "--json", ]); diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx index 032871053..6d9d2a037 100644 --- a/src/handlers/eval/batch-insights/run/index.tsx +++ b/src/handlers/eval/batch-insights/run/index.tsx @@ -17,7 +17,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => ...SessionSource.flags, flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT])), flag( - "evaluator", + "evaluators", "optional evaluator ID(s) to run alongside the insights", z.array(z.string()).optional(), ), @@ -36,7 +36,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => name: flags["name"], description: flags["description"], insightIds: flags["insight"], - evaluatorIds: flags["evaluator"], + evaluatorIds: flags["evaluators"], source, kmsKeyArn: flags["kms-key-arn"], }, diff --git a/src/handlers/eval/ondemand/evaluate/index.tsx b/src/handlers/eval/ondemand/evaluate/index.tsx index d9bdeea1f..4d50ca7b2 100644 --- a/src/handlers/eval/ondemand/evaluate/index.tsx +++ b/src/handlers/eval/ondemand/evaluate/index.tsx @@ -19,7 +19,7 @@ export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => z.string().optional(), ), flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()), - flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), flag( "lookback-days", "time filter: evaluate sessions from the last N days", @@ -51,9 +51,9 @@ export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => if (!flags["agent"]) { throw new InputValidationError("on-demand requires '--agent'"); } - if (!flags["evaluator"] || flags["evaluator"].length === 0) { + if (!flags["evaluators"] || flags["evaluators"].length === 0) { throw new InputValidationError( - "required option '--evaluator ' not specified", + "required option '--evaluators ' not specified", ); } @@ -84,7 +84,7 @@ export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => ); const result = await core.eval.evaluate( - { traces, evaluatorIds: flags["evaluator"], groundTruth }, + { traces, evaluatorIds: flags["evaluators"], groundTruth }, opts, ); ctx.require(JsonRendererKey).renderJson(result); diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx index b1aefddc1..b309340b7 100644 --- a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -61,7 +61,7 @@ describe("eval ondemand evaluate (fixture-backed)", () => { WINDOW_START, "--end-time", WINDOW_END, - "--evaluator", + "--evaluators", "Builtin.Helpfulness", ]); @@ -113,7 +113,7 @@ describe("eval ondemand evaluate (fixture-backed)", () => { '{"prompt":"{input}"}', "--dataset", SIMULATE_DATASET, - "--evaluator", + "--evaluators", "Builtin.Correctness", "--ingestion-wait-ms", isRecording() ? "150000" : "0", diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx index 58991327d..56892a454 100644 --- a/src/handlers/eval/ondemand/ondemand.test.tsx +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -141,7 +141,7 @@ async function runWithRealCore(options: LogsOptions, logger = createSilentLogger "evaluate", "--agent", RUNTIME_ID, - "--evaluator", + "--evaluators", "Builtin.Helpfulness", "--session-ids", "session-1", @@ -163,7 +163,7 @@ const BASE = [ "evaluate", "--agent", "a-1", - "--evaluator", + "--evaluators", "Builtin.Helpfulness", ]; @@ -195,22 +195,22 @@ describe("eval ondemand simulate", () => { '{"prompt":"{input}"}', "--dataset", "/tmp/ds.jsonl", - "--evaluator", + "--evaluators", "Builtin.Helpfulness", ]; test.each<[RegExp, string[]]>([ [ /--runtime-id/, - ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluators", "E"], ], [ /--payload-template/, - ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"], + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluators", "E"], ], - [/--dataset/, ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E"]], + [/--dataset/, ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluators", "E"]], [ - /--evaluator/, + /--evaluators/, ["--runtime-id", "r-1", "--payload-template", "{}", "--dataset", "/tmp/ds.jsonl"], ], ])("rejects when a required flag is missing (%s)", async (expected, args) => { @@ -280,13 +280,21 @@ describe("eval ondemand evaluate validation", () => { test.each<[string, string[], RegExp]>([ [ "requires --agent", - ["eval", "ondemand", "evaluate", "--evaluator", "Builtin.Helpfulness", "--session-ids", "s1"], + [ + "eval", + "ondemand", + "evaluate", + "--evaluators", + "Builtin.Helpfulness", + "--session-ids", + "s1", + ], /--agent/, ], [ - "requires --evaluator", + "requires --evaluators", ["eval", "ondemand", "evaluate", "--agent", "a-1", "--session-ids", "s1"], - /--evaluator/, + /--evaluators/, ], ["rejects an empty session source", BASE, /session source/], [ diff --git a/src/handlers/eval/ondemand/simulate/index.tsx b/src/handlers/eval/ondemand/simulate/index.tsx index 3e1ee7f6e..7284666e3 100644 --- a/src/handlers/eval/ondemand/simulate/index.tsx +++ b/src/handlers/eval/ondemand/simulate/index.tsx @@ -34,7 +34,7 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => flag("user-id", "Runtime user ID", z.string().optional()), flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()), flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()), - flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()), + flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()), flag( "ingestion-wait-ms", "ms to wait for span ingestion before grading (default 180000; 0 to skip)", @@ -49,16 +49,16 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) => } if (!flags["dataset"]) throw new InputValidationError("required option '--dataset' not specified"); - if (!flags["evaluator"]?.length) { + if (!flags["evaluators"]?.length) { throw new InputValidationError( - "required option '--evaluator ' not specified", + "required option '--evaluators ' not specified", ); } const runtimeId = flags["runtime-id"]; const payloadTemplate = flags["payload-template"]; const dataset = flags["dataset"]; - const evaluatorIds = flags["evaluator"]; + const evaluatorIds = flags["evaluators"]; await withUserCancellation(async (signal) => { const opts = coreOptsFromCtx(ctx); diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index c5f60f204..7dc88f4c6 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -24,7 +24,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => "the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin), as an alternative to --agent", z.string().optional(), ), - flag("evaluator", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()), + flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()), flag( "sampling-rate", "percentage of sessions to sample (0.01-100)", @@ -64,9 +64,9 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => "required option '--sampling-rate ' not specified", ); } - if (!flags["evaluator"] || flags["evaluator"].length === 0) { + if (!flags["evaluators"] || flags["evaluators"].length === 0) { throw new InputValidationError( - "required option '--evaluator ' not specified", + "required option '--evaluators ' not specified", ); } @@ -91,7 +91,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => "filters", await source.resolveText("filters", flags["filters"]), ), - evaluatorIds: flags["evaluator"], + evaluatorIds: flags["evaluators"], evaluationExecutionRoleArn: flags["role-arn"], enableOnCreate: flags["enable-on-create"] === undefined diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index c56bfb4a9..a749fd103 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -131,7 +131,7 @@ describe("online-eval CRUDL", () => { CONFIG_NAME, "--agent", FIXTURE_AGENT_ID, - "--evaluator", + "--evaluators", FIXTURE_EVALUATOR_ID, "--sampling-rate", "10", @@ -252,7 +252,7 @@ describe("flag validation", () => { "create", "--name", CONFIG_NAME, - "--evaluator", + "--evaluators", FIXTURE_EVALUATOR_ID, "--sampling-rate", "10", @@ -274,7 +274,7 @@ describe("flag validation", () => { FIXTURE_AGENT_ID, "--data-source-config", '{"cloudWatchLogs":{"logGroupNames":["/custom"],"serviceNames":["svc"]}}', - "--evaluator", + "--evaluators", FIXTURE_EVALUATOR_ID, "--sampling-rate", "10", @@ -296,7 +296,7 @@ describe("flag validation", () => { '{"cloudWatchLogs":{"logGroupNames":["/custom"],"serviceNames":["svc"]}}', "--endpoint", "prod", - "--evaluator", + "--evaluators", FIXTURE_EVALUATOR_ID, "--sampling-rate", "10", @@ -316,7 +316,7 @@ describe("flag validation", () => { CONFIG_NAME, "--data-source-config", "not-json", - "--evaluator", + "--evaluators", FIXTURE_EVALUATOR_ID, "--sampling-rate", "10", @@ -406,9 +406,9 @@ describe("execution role KMS scoping", () => { // Builtin.Correctness is backed by the hand-authored fixture carrying a // kmsKeyArn; Builtin.Helpfulness carries none, so this covers both arms of // the resolution in one create. - "--evaluator", + "--evaluators", FIXTURE_EVALUATOR_ID, - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "10", @@ -442,7 +442,7 @@ describe("execution role scoping on update", () => { WARN_CONFIG_NAME, "--agent", FIXTURE_AGENT_ID, - "--evaluator", + "--evaluators", FIXTURE_EVALUATOR_ID, "--sampling-rate", "10", diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 93b638abc..b2f487ce6 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -30,7 +30,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => z.string().optional(), ), flag( - "evaluator", + "evaluators", "the ID(s) of the evaluators to apply (replaces the existing list)", z.array(z.string()).optional(), ), @@ -88,7 +88,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => "filters", await source.resolveText("filters", flags["filters"]), ), - evaluatorIds: flags["evaluator"], + evaluatorIds: flags["evaluators"], agent: flags["agent"], endpoint: flags["endpoint"], clearEndpoint: flags["clear-endpoint"] === "true", diff --git a/src/handlers/project/add/online-eval/index.test.ts b/src/handlers/project/add/online-eval/index.test.ts index 97dc789d9..9967c4b1a 100644 --- a/src/handlers/project/add/online-eval/index.test.ts +++ b/src/handlers/project/add/online-eval/index.test.ts @@ -65,7 +65,7 @@ describe("project add online-eval", () => { "x", "--agent", "agent_python_minimal", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "50", @@ -79,7 +79,7 @@ describe("project add online-eval", () => { "x", "--log-group-name", "/aws/foo", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "50", @@ -95,7 +95,7 @@ describe("project add online-eval", () => { "agent_python_minimal", "--endpoint", "PROD", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "10", @@ -111,7 +111,7 @@ describe("project add online-eval", () => { "/aws/foo", "--service-name", "svc", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "25", @@ -125,7 +125,7 @@ describe("project add online-eval", () => { "x", "--log-group-name", "/aws/foo", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "5", @@ -141,7 +141,7 @@ describe("project add online-eval", () => { "x", "--agent", "agent_python_minimal", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "5", @@ -157,7 +157,7 @@ describe("project add online-eval", () => { "x", "--log-group-name", "/aws/foo", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "5", @@ -182,7 +182,7 @@ describe("project add online-eval", () => { "x", "--log-group-name", "/aws/foo", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "50", @@ -209,7 +209,7 @@ describe("project add online-eval", () => { "x", "--agent", "a", - "--evaluator", + "--evaluators", "e", "--sampling-rate", "5", @@ -218,8 +218,8 @@ describe("project add online-eval", () => { }); test.each<[string, string[]]>([ - ["missing --name", ["--log-group-name", "/x", "--evaluator", "e", "--sampling-rate", "10"]], - ["missing --sampling-rate", ["--name", "x", "--log-group-name", "/x", "--evaluator", "e"]], + ["missing --name", ["--log-group-name", "/x", "--evaluators", "e", "--sampling-rate", "10"]], + ["missing --sampling-rate", ["--name", "x", "--log-group-name", "/x", "--evaluators", "e"]], [ "--agent and --log-group-name are mutually exclusive", [ @@ -229,7 +229,7 @@ describe("project add online-eval", () => { "a", "--log-group-name", "/x", - "--evaluator", + "--evaluators", "e", "--sampling-rate", "10", @@ -237,7 +237,7 @@ describe("project add online-eval", () => { ], [ "neither --agent nor --log-group-name", - ["--name", "x", "--evaluator", "e", "--sampling-rate", "10"], + ["--name", "x", "--evaluators", "e", "--sampling-rate", "10"], ], ["no evaluator", ["--name", "x", "--agent", "a", "--sampling-rate", "10"]], [ @@ -249,7 +249,7 @@ describe("project add online-eval", () => { "/x", "--endpoint", "PROD", - "--evaluator", + "--evaluators", "e", "--sampling-rate", "10", @@ -264,7 +264,7 @@ describe("project add online-eval", () => { "a", "--service-name", "svc", - "--evaluator", + "--evaluators", "e", "--sampling-rate", "10", diff --git a/src/handlers/project/add/online-eval/index.ts b/src/handlers/project/add/online-eval/index.ts index a1a18740c..5ca5f16a1 100644 --- a/src/handlers/project/add/online-eval/index.ts +++ b/src/handlers/project/add/online-eval/index.ts @@ -33,7 +33,7 @@ export const createAddOnlineEvalHandler = (config: AddProjectResourceConfig) => z.array(z.string()).optional(), ), flag( - "evaluator", + "evaluators", "evaluator name(s), Builtin.* IDs, or ARNs to apply", z.array(z.string()).optional(), ), @@ -68,7 +68,7 @@ export const createAddOnlineEvalHandler = (config: AddProjectResourceConfig) => endpoint: flags["endpoint"], logGroupNames: flags["log-group-name"], serviceNames: flags["service-name"], - evaluators: flags["evaluator"], + evaluators: flags["evaluators"], samplingRate: flags["sampling-rate"], description: flags["description"], enableOnCreate: diff --git a/src/handlers/project/remove/index.test.ts b/src/handlers/project/remove/index.test.ts index e691ceb83..c194161bf 100644 --- a/src/handlers/project/remove/index.test.ts +++ b/src/handlers/project/remove/index.test.ts @@ -137,7 +137,7 @@ describe("project remove", () => { "quality", "--agent", "agent_python_minimal", - "--evaluator", + "--evaluators", "Builtin.Correctness", "--sampling-rate", "5",