diff --git a/packages/runtime-core/src/command-registry.ts b/packages/runtime-core/src/command-registry.ts index f23858b8..b11fb71e 100644 --- a/packages/runtime-core/src/command-registry.ts +++ b/packages/runtime-core/src/command-registry.ts @@ -1304,7 +1304,9 @@ export const commandRegistry = [ { name: "presentation-url", description: "Optional frontend URL corresponding to the edited content. Enables bounded frontend-to-editor presentation evidence.", format: "path or URL" }, { name: "presentation-frontend-selector", description: "Optional frontend content selector for presentation evidence; defaults to body and is bounded to 512 characters.", format: "CSS selector" }, { name: "presentation-editor-selector", description: "Optional editor canvas selector for presentation evidence; defaults to the block list layout and is bounded to 512 characters.", format: "CSS selector" }, - { name: "presentation-threshold", description: "Maximum geometry delta ratio accepted by presentation evidence; defaults to 0.02.", format: "number between 0 and 1" }, + { name: "presentation-pixel-threshold", description: "Maximum pixelmatch per-pixel color-distance threshold; defaults to 0.02.", format: "number between 0 and 1" }, + { name: "presentation-overlap-mismatch-threshold", description: "Maximum mismatched-pixel ratio across the shared canvas area; defaults to 0.1.", format: "number between 0 and 1" }, + { name: "presentation-dimension-drift-threshold", description: "Maximum canvas-dimension delta ratio; defaults to 0.01.", format: "number between 0 and 1" }, { name: "capture", description: "Comma-separated artifacts to capture after opening the editor.", format: "steps,console,errors,html,screenshot,editor-state,editor-validity" }, { name: "artifact-prefix", description: "Optional artifact directory relative to the runtime artifact root for this invocation; defaults to files/browser. Use files/browser/editor-open/ to isolate per-fixture editor-open evidence in a batch.", format: "relative artifact directory" }, ], diff --git a/packages/runtime-playground/src/browser-artifacts.ts b/packages/runtime-playground/src/browser-artifacts.ts index 2e7a3f60..9703df34 100644 --- a/packages/runtime-playground/src/browser-artifacts.ts +++ b/packages/runtime-playground/src/browser-artifacts.ts @@ -316,6 +316,8 @@ export interface BrowserEditorPresentationMatchSummary { diagnostic?: string equivalentCanvasWidths?: boolean majorGeometryDrift?: boolean + majorVisualDivergence?: boolean + comparison?: BrowserEditorPresentationComparison unreadableContent?: boolean hiddenContent?: boolean unresolvedAssetCount?: number @@ -329,6 +331,20 @@ export interface BrowserEditorPresentationMatchSummary { } } +export interface BrowserEditorPresentationComparison { + pixelThreshold: number + overlapMismatchThreshold: number + dimensionDriftThreshold: number + mismatchPixels: number + totalPixels: number + mismatchRatio: number + overlapMismatchPixels: number + overlapPixels: number + overlapMismatchRatio: number + dimensionDeltaPixels: number + dimensionDeltaRatio: number +} + export interface BrowserEditorPresentationSurfaceGeometry { width: number height: number diff --git a/packages/runtime-playground/src/editor-command-runners.ts b/packages/runtime-playground/src/editor-command-runners.ts index 73ab950d..abe80bbb 100644 --- a/packages/runtime-playground/src/editor-command-runners.ts +++ b/packages/runtime-playground/src/editor-command-runners.ts @@ -1040,7 +1040,7 @@ async function captureEditorPresentationMatch(input: { if (!presentationUrl) return { schema: "wp-codebox/editor-presentation-match/v1", status: "unavailable", diagnostic: "presentation-url was not supplied" } const frontendSelector = boundedPresentationSelector(input.args, "presentation-frontend-selector", "body") const editorSelector = boundedPresentationSelector(input.args, "presentation-editor-selector", EDITOR_CANVAS_DEFAULT_LAYOUT_SELECTOR) - const threshold = presentationThreshold(input.args) + const thresholds = presentationThresholds(input.args) const frontendPath = input.artifactSession.absolutePath("presentation-frontend.png") const editorPath = input.artifactSession.absolutePath("presentation-editor.png") const frontendRef = input.artifactSession.path("presentation-frontend.png") @@ -1086,18 +1086,43 @@ async function captureEditorPresentationMatch(input: { await input.artifactSession.writeGenerated("screenshot", "presentation-editor.png", async (path) => { await editor.screenshot({ path, timeout: input.waitTimeoutMs }); }) let comparison: Awaited> | undefined await input.artifactSession.writeGenerated("screenshot", "presentation-diff.png", async (path) => { - comparison = await comparePngFiles(frontendPath, editorPath, path, { threshold, includeAA: false, maxRegions: 8 }) + comparison = await comparePngFiles(frontendPath, editorPath, path, { threshold: thresholds.pixel, includeAA: false, maxRegions: 8 }) }) if (!comparison) throw new Error("Presentation comparison did not produce metrics") const equivalentCanvasWidths = comparison.source.width === comparison.candidate.width - // The same bounded threshold applies to canvas extent and the shared - // visual region; equal widths alone must not certify different renders. - const majorGeometryDrift = comparison.dimensionDeltaRatio > threshold || comparison.overlapMismatchRatio > threshold + const majorGeometryDrift = comparison.dimensionDeltaRatio > thresholds.dimensionDrift + const majorVisualDivergence = comparison.overlapMismatchRatio > thresholds.overlapMismatch const unreadableContent = !frontendEvidence.readable || !editorEvidence.readable const hiddenContent = !frontendEvidence.visible || !editorEvidence.visible const unresolvedAssetCount = frontendEvidence.unresolvedAssets + editorEvidence.unresolvedAssets - const passed = equivalentCanvasWidths && !majorGeometryDrift && !unreadableContent && !hiddenContent && unresolvedAssetCount === 0 - return { schema: "wp-codebox/editor-presentation-match/v1", status: passed ? "passed" : "failed", equivalentCanvasWidths, majorGeometryDrift, unreadableContent, hiddenContent, unresolvedAssetCount, frontendScreenshot: frontendRef, editorScreenshot: editorRef, diffScreenshot: diffRef, geometry: { frontend: frontendGeometry, liveEditor: liveEditorGeometry, isolatedEditor: isolatedEditorGeometry } } + const passed = equivalentCanvasWidths && !majorGeometryDrift && !majorVisualDivergence && !unreadableContent && !hiddenContent && unresolvedAssetCount === 0 + return { + schema: "wp-codebox/editor-presentation-match/v1", + status: passed ? "passed" : "failed", + equivalentCanvasWidths, + majorGeometryDrift, + majorVisualDivergence, + comparison: { + pixelThreshold: thresholds.pixel, + overlapMismatchThreshold: thresholds.overlapMismatch, + dimensionDriftThreshold: thresholds.dimensionDrift, + mismatchPixels: comparison.mismatchPixels, + totalPixels: comparison.totalPixels, + mismatchRatio: comparison.mismatchRatio, + overlapMismatchPixels: comparison.overlapMismatchPixels, + overlapPixels: comparison.overlapPixels, + overlapMismatchRatio: comparison.overlapMismatchRatio, + dimensionDeltaPixels: comparison.dimensionDeltaPixels, + dimensionDeltaRatio: comparison.dimensionDeltaRatio, + }, + unreadableContent, + hiddenContent, + unresolvedAssetCount, + frontendScreenshot: frontendRef, + editorScreenshot: editorRef, + diffScreenshot: diffRef, + geometry: { frontend: frontendGeometry, liveEditor: liveEditorGeometry, isolatedEditor: isolatedEditorGeometry }, + } } finally { await frontendContext.close() } @@ -1227,11 +1252,19 @@ function boundedPresentationSelector(args: string[], name: string, fallback: str return selector } -function presentationThreshold(args: string[]): number { - const raw = argValue(args, "presentation-threshold")?.trim() - if (!raw) return 0.02 +function presentationThresholds(args: string[]): { pixel: number; overlapMismatch: number; dimensionDrift: number } { + return { + pixel: boundedPresentationThreshold(args, "presentation-pixel-threshold", 0.02), + overlapMismatch: boundedPresentationThreshold(args, "presentation-overlap-mismatch-threshold", 0.1), + dimensionDrift: boundedPresentationThreshold(args, "presentation-dimension-drift-threshold", 0.01), + } +} + +function boundedPresentationThreshold(args: string[], name: string, fallback: number): number { + const raw = argValue(args, name)?.trim() + if (!raw) return fallback const value = Number(raw) - if (!Number.isFinite(value) || value < 0 || value > 1) throw new Error(`wordpress.editor-open presentation-threshold must be between 0 and 1: ${raw}`) + if (!Number.isFinite(value) || value < 0 || value > 1) throw new Error(`wordpress.editor-open ${name} must be between 0 and 1: ${raw}`) return value } diff --git a/tests/browser-routed-command-security.test.ts b/tests/browser-routed-command-security.test.ts index 5400b069..1cc047c4 100644 --- a/tests/browser-routed-command-security.test.ts +++ b/tests/browser-routed-command-security.test.ts @@ -27,6 +27,12 @@ const GROWING_PRESENTATION_IDENTITIES = ["3".repeat(64), "4".repeat(64)] const DELAYED_POST_PRESENTATION_IDENTITY = "5".repeat(64) const EDITOR_PRESENTATION_CONTRACT_MARKER = "WP_CODEBOX_EDITOR_PRESENTATION_CONTRACT:" const matchedPresentationMarkup = `
Matched presentation
` +const boundedRasterizationFrontendMarkup = `
Bounded rasterization
` +const boundedRasterizationEditorMarkup = `
Bounded rasterization
` +const majorVisualFrontendMarkup = `
Major visual divergence
` +const majorVisualEditorMarkup = `
Major visual divergence
` +const majorGeometryFrontendMarkup = `
Major geometry divergence
` +const majorGeometryEditorMarkup = `
Major geometry divergence
` const editorShell = `` const delayedCanvasEditorHtml = `${editorShell}
Transition
` @@ -76,6 +85,18 @@ test("real browser commands sanitize console, artifacts, stdout, and failure std ? "
Broken editor fixture
" : request.url?.startsWith("/presentation") ? "
Deliberately different frontend fixture
" + : request.url?.startsWith("/bounded-rasterization-frontend") + ? boundedRasterizationFrontendMarkup + : request.url?.startsWith("/bounded-rasterization-editor") + ? boundedRasterizationEditorHtml + : request.url?.startsWith("/major-visual-frontend") + ? majorVisualFrontendMarkup + : request.url?.startsWith("/major-visual-editor") + ? majorVisualEditorHtml + : request.url?.startsWith("/major-geometry-frontend") + ? majorGeometryFrontendMarkup + : request.url?.startsWith("/major-geometry-editor") + ? majorGeometryEditorHtml : request.url?.startsWith("/matched-frontend") ? matchedPresentationMarkup : request.url?.startsWith("/matched-editor") @@ -158,13 +179,14 @@ test("real browser commands sanitize console, artifacts, stdout, and failure std server, spec: { command: "wordpress.editor-open", args: [`url=${PUBLIC_URL}`, "presentation-url=/presentation", "route-host=routed.test", "capture=steps", "wait-timeout=5s"] }, }) - const output = JSON.parse(result.output) as { summary: { editorPresentation: { matchedRendering: { status: string; frontendScreenshot: string; editorScreenshot: string; diffScreenshot: string; equivalentCanvasWidths: boolean; majorGeometryDrift: boolean; unreadableContent: boolean; hiddenContent: boolean; unresolvedAssetCount: number } } } } - const { geometry, ...matchedRendering } = output.summary.editorPresentation.matchedRendering as typeof output.summary.editorPresentation.matchedRendering & { geometry: { frontend: { childCount: number }; liveEditor: { childCount: number }; isolatedEditor: { childCount: number } } } + const output = JSON.parse(result.output) as { summary: { editorPresentation: { matchedRendering: { status: string; frontendScreenshot: string; editorScreenshot: string; diffScreenshot: string; equivalentCanvasWidths: boolean; majorGeometryDrift: boolean; majorVisualDivergence: boolean; unreadableContent: boolean; hiddenContent: boolean; unresolvedAssetCount: number } } } } + const { geometry, comparison, ...matchedRendering } = output.summary.editorPresentation.matchedRendering as typeof output.summary.editorPresentation.matchedRendering & { comparison: Record; geometry: { frontend: { childCount: number }; liveEditor: { childCount: number }; isolatedEditor: { childCount: number } } } assert.deepEqual(matchedRendering, { schema: "wp-codebox/editor-presentation-match/v1", status: "failed", equivalentCanvasWidths: true, - majorGeometryDrift: true, + majorGeometryDrift: false, + majorVisualDivergence: true, unreadableContent: false, hiddenContent: false, unresolvedAssetCount: 0, @@ -172,6 +194,10 @@ test("real browser commands sanitize console, artifacts, stdout, and failure std editorScreenshot: "files/browser/presentation-editor.png", diffScreenshot: "files/browser/presentation-diff.png", }) + assert.deepEqual(Object.keys(comparison).sort(), ["dimensionDeltaPixels", "dimensionDeltaRatio", "dimensionDriftThreshold", "mismatchPixels", "mismatchRatio", "overlapMismatchPixels", "overlapMismatchRatio", "overlapMismatchThreshold", "overlapPixels", "pixelThreshold", "totalPixels"].sort()) + assert.equal(comparison.pixelThreshold, 0.02) + assert.equal(comparison.overlapMismatchThreshold, 0.1) + assert.equal(comparison.dimensionDriftThreshold, 0.01) assert.equal(geometry.frontend.childCount, 1) assert.equal(geometry.liveEditor.childCount, 1) assert.equal(geometry.isolatedEditor.childCount, 1) @@ -190,6 +216,37 @@ test("real browser commands sanitize console, artifacts, stdout, and failure std assert.equal(output.summary.editorPresentation.matchedRendering.status, "passed") }) + await withTempDir("wp-codebox-editor-presentation-thresholds-", async (artifactRoot) => { + const runPresentation = async (url: string, presentationUrl: string) => { + const result = await runEditorOpenCommand({ + artifactRoot, + runPlaygroundCommand, + runtimeSpec, + server, + spec: { command: "wordpress.editor-open", args: [`url=http://routed.test/${url}`, `presentation-url=/${presentationUrl}`, "presentation-frontend-selector=.block-editor-block-list__layout", "route-host=routed.test", "capture=steps", "wait-timeout=5s"] }, + }) + return (JSON.parse(result.output) as { summary: { editorPresentation: { matchedRendering: { status: string; majorGeometryDrift: boolean; majorVisualDivergence: boolean; comparison: { overlapMismatchRatio: number; dimensionDeltaRatio: number } } } } }).summary.editorPresentation.matchedRendering + } + + const boundedRasterization = await runPresentation("bounded-rasterization-editor", "bounded-rasterization-frontend") + assert.equal(boundedRasterization.status, "passed") + assert.equal(boundedRasterization.majorGeometryDrift, false) + assert.equal(boundedRasterization.majorVisualDivergence, false) + assert.ok(boundedRasterization.comparison.dimensionDeltaRatio > 0 && boundedRasterization.comparison.dimensionDeltaRatio < 0.01, JSON.stringify(boundedRasterization.comparison)) + assert.ok(boundedRasterization.comparison.overlapMismatchRatio > 0 && boundedRasterization.comparison.overlapMismatchRatio < 0.1) + + const majorVisual = await runPresentation("major-visual-editor", "major-visual-frontend") + assert.equal(majorVisual.status, "failed") + assert.equal(majorVisual.majorGeometryDrift, false) + assert.equal(majorVisual.majorVisualDivergence, true) + + const majorGeometry = await runPresentation("major-geometry-editor", "major-geometry-frontend") + assert.equal(majorGeometry.status, "failed") + assert.equal(majorGeometry.majorGeometryDrift, true) + assert.equal(majorGeometry.majorVisualDivergence, false) + assert.ok(majorGeometry.comparison.dimensionDeltaRatio > 0.01) + }) + await withTempDir("wp-codebox-editor-onboarding-preference-", async (artifactRoot) => { const result = await runEditorOpenCommand({ artifactRoot,