From 9b7547cae0a8bc175d9c1e257c4e81c2de551891 Mon Sep 17 00:00:00 2001 From: guaguagf <99520715+guaguagf@users.noreply.github.com> Date: Sat, 15 Aug 2026 07:45:58 +0800 Subject: [PATCH 1/5] docs(test): document decisive frontend assertions Clarify assertion guidelines in skill-template.md --- docs/cli-v1-agent-install/skill-template.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/cli-v1-agent-install/skill-template.md b/docs/cli-v1-agent-install/skill-template.md index 764e04e..bf0e1ba 100644 --- a/docs/cli-v1-agent-install/skill-template.md +++ b/docs/cli-v1-agent-install/skill-template.md @@ -173,6 +173,10 @@ blue 'Submit' button"`. The agent does its own semantic match; a guessed label - **1–2 assertions, at the end, on content existence** rather than UI copy / formatting. (Quoting a literal you submitted earlier in the same plan, to verify it round-trips, is fine.) +- **Keep each assertion single and decisive.** Avoid conditional or multi-branch + wording such as "either A or B", "if A then B", `unless`, or `whether` — it can + exhaust the frontend run budget before a verdict is emitted. `test lint` warns + on these patterns before the plan consumes run credits. - **Assertion targets name the specific page region** (panel / tab / output area). Otherwise the agent settles for "some element with that text is visible anywhere," which passes for wrong reasons. From 063e825ba1bf1a075b45abf40dc5681235eef99b Mon Sep 17 00:00:00 2001 From: guaguagf <99520715+guaguagf@users.noreply.github.com> Date: Sat, 15 Aug 2026 07:50:09 +0800 Subject: [PATCH 2/5] feat(test): warn on conditional frontend assertions --- src/commands/test.ts | 58 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/src/commands/test.ts b/src/commands/test.ts index 198ce66..cdf3c21 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -4742,6 +4742,42 @@ export interface CliLintReport { checked: number; valid: number; issues: CliLintIssue[]; + /** Non-fatal plan-quality findings; omitted when there are none. */ + warnings?: CliLintIssue[]; +} + +/** + * Conditional and multi-branch assertion wording makes the frontend agent + * explore several outcomes and can exhaust its step budget before it emits a + * verdict. This is deliberately a narrow heuristic: only assertion steps are + * considered, and the finding stays non-fatal because the plan is structurally + * valid and may still be intentional. + */ +const CONDITIONAL_ASSERTION_PATTERN = + /\b(?:either|or|otherwise|unless|whether)\b|\bif\b[^.!?]*\bthen\b/i; + +function collectAssertionComplexityWarnings( + parsed: unknown, + prefix = '', +): Array<{ field: string; reason: string }> { + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return []; + const obj = parsed as Record; + if (obj.type !== undefined && obj.type !== 'frontend') return []; + if (!Array.isArray(obj.planSteps)) return []; + + const warnings: Array<{ field: string; reason: string }> = []; + obj.planSteps.forEach((step, index) => { + if (typeof step !== 'object' || step === null || Array.isArray(step)) return; + const candidate = step as Record; + if (candidate.type !== 'assertion' || typeof candidate.description !== 'string') return; + if (!CONDITIONAL_ASSERTION_PATTERN.test(candidate.description)) return; + warnings.push({ + field: `${prefix}planSteps[${index}].description`, + reason: + 'uses conditional or multi-branch wording; choose one observable outcome and write a single, decisive assertion to avoid exhausting the frontend run budget', + }); + }); + return warnings; } /** @@ -4795,6 +4831,7 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { @@ -4809,6 +4846,10 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { @@ -4823,6 +4864,9 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise issue.file)).size; - const report: CliLintReport = { checked, valid: checked - filesWithIssues, issues }; + const report: CliLintReport = { + checked, + valid: checked - filesWithIssues, + issues, + ...(warnings.length > 0 ? { warnings } : {}), + }; out.print(report, () => [ ...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`), - `${report.valid}/${report.checked} valid, ${issues.length} problem(s)`, + ...warnings.map(warning => `[warning] ${warning.file}: ${warning.field}: ${warning.reason}`), + `${report.valid}/${report.checked} valid, ${issues.length} problem(s)` + + (warnings.length > 0 ? `, ${warnings.length} warning(s)` : ''), ].join('\n'), ); if (issues.length > 0) { From 92ce7dc470df4c951bc48111544c5ecb3afad714 Mon Sep 17 00:00:00 2001 From: guaguagf <99520715+guaguagf@users.noreply.github.com> Date: Sat, 15 Aug 2026 08:15:40 +0800 Subject: [PATCH 3/5] test(test): cover conditional assertion warnings --- src/commands/test.lint-warning.spec.ts | 67 ++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/commands/test.lint-warning.spec.ts diff --git a/src/commands/test.lint-warning.spec.ts b/src/commands/test.lint-warning.spec.ts new file mode 100644 index 0000000..3cd67ef --- /dev/null +++ b/src/commands/test.lint-warning.spec.ts @@ -0,0 +1,67 @@ +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { runLint } from './test.js'; + +describe('runLint assertion-complexity warnings', () => { + it('warns when a frontend assertion contains conditional or multi-branch wording', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-conditional-')); + const file = join(dir, 'plan.json'); + writeFileSync( + file, + JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Knowledge Web renders', + planSteps: [ + { + type: 'assertion', + description: 'Verify either an interactive graph canvas or a clear empty-state message', + }, + ], + }), + 'utf8', + ); + + const report = await runLint( + { profile: 'default', output: 'json', debug: false, planFrom: file }, + { stdout: () => undefined }, + ); + + expect(report).toMatchObject({ checked: 1, valid: 1, issues: [] }); + expect(report.warnings).toEqual([ + expect.objectContaining({ + field: 'planSteps[0].description', + reason: expect.stringContaining('single, decisive assertion'), + }), + ]); + }); + + it('does not flag actions or single-outcome assertions', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-decisive-')); + const file = join(dir, 'plan.json'); + writeFileSync( + file, + JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Checkout works', + planSteps: [ + { type: 'action', description: 'Open the cart or return to the catalog' }, + { type: 'assertion', description: 'Verify the order total is visible' }, + ], + }), + 'utf8', + ); + + const report = await runLint( + { profile: 'default', output: 'json', debug: false, planFrom: file }, + { stdout: () => undefined }, + ); + + expect(report).toEqual({ checked: 1, valid: 1, issues: [] }); + }); +}); From 05457551a0be2469a44c5eccd90ad97a538fb082 Mon Sep 17 00:00:00 2001 From: guaguagf <99520715+guaguagf@users.noreply.github.com> Date: Sat, 15 Aug 2026 08:32:25 +0800 Subject: [PATCH 4/5] fix(test): keep lint advisories off stdout Add warning output to stderr when warnings are present. --- src/commands/test.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/commands/test.ts b/src/commands/test.ts index cdf3c21..b3c46ad 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -4940,12 +4940,17 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise 0 ? { warnings } : {}), }; + if (opts.output === 'text' && warnings.length > 0) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + for (const warning of warnings) { + stderr(`[warning] ${warning.file}: ${warning.field}: ${warning.reason}`); + } + stderr(`${warnings.length} warning(s)`); + } out.print(report, () => [ ...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`), - ...warnings.map(warning => `[warning] ${warning.file}: ${warning.field}: ${warning.reason}`), - `${report.valid}/${report.checked} valid, ${issues.length} problem(s)` + - (warnings.length > 0 ? `, ${warnings.length} warning(s)` : ''), + `${report.valid}/${report.checked} valid, ${issues.length} problem(s)`, ].join('\n'), ); if (issues.length > 0) { From ffbb942b0bfec511eb520d6f9663dfe9759c988b Mon Sep 17 00:00:00 2001 From: guaguagf <99520715+guaguagf@users.noreply.github.com> Date: Sat, 15 Aug 2026 08:36:22 +0800 Subject: [PATCH 5/5] test(test): cover stderr lint warnings Add test to ensure text warnings are machine-safe on stderr and stdout. --- src/commands/test.lint-warning.spec.ts | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/commands/test.lint-warning.spec.ts b/src/commands/test.lint-warning.spec.ts index 3cd67ef..dc0620a 100644 --- a/src/commands/test.lint-warning.spec.ts +++ b/src/commands/test.lint-warning.spec.ts @@ -40,6 +40,38 @@ describe('runLint assertion-complexity warnings', () => { ]); }); + it('keeps text warnings on stderr and stdout machine-safe', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-warning-streams-')); + const file = join(dir, 'plan.json'); + writeFileSync( + file, + JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Knowledge Web renders', + planSteps: [ + { + type: 'assertion', + description: 'Verify either an interactive graph canvas or a clear empty-state message', + }, + ], + }), + 'utf8', + ); + const stdout: string[] = []; + const stderr: string[] = []; + + await runLint( + { profile: 'default', output: 'text', debug: false, planFrom: file }, + { stdout: line => stdout.push(line), stderr: line => stderr.push(line) }, + ); + + expect(stdout.join('\n')).toBe('1/1 valid, 0 problem(s)'); + expect(stdout.join('\n')).not.toContain('[warning]'); + expect(stderr.join('\n')).toContain('[warning]'); + expect(stderr.at(-1)).toBe('1 warning(s)'); + }); + it('does not flag actions or single-outcome assertions', async () => { const dir = mkdtempSync(join(tmpdir(), 'cli-lint-decisive-')); const file = join(dir, 'plan.json');