From 0c39e5e34001fbddb80b99a839ea0373c57c814f Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:59:33 +0100 Subject: [PATCH 1/2] Let a sparse page vouch for itself with data-renderdiff-rendered A one-viewport fixture page with a few dozen chars of text was refused as "page did not render": the gate can't tell a page sparse on purpose from one that never mounted. A visible element marked data-renderdiff-rendered now counts as rendered content, and both refusals (run and probe) name the threshold, the marker, and where it must not go. Fixes #100 Co-Authored-By: Claude Opus 5.5 --- docs/how-it-works.md | 3 ++- docs/scoping.md | 5 +++++ skills/renderdiff/reference.md | 3 ++- src/capture.ts | 4 +++- src/probe.ts | 4 ++-- src/run.ts | 6 +++++- tests/visual/guardrails.test.ts | 14 +++++++++++++- 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 7741bcc..7311a77 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -8,7 +8,8 @@ honestly. This is what renderdiff does to make sure of that. - **Refuses a shot** it can't trust: an HTTP status of 400 or above on either side is an error naming the status. A route where *both* sides look blank — under 200 characters of text and no visible images, SVG, canvas, video, iframes or form controls — errors instead - of quietly scoring as a match. If only one side is blank, the diff shows it. + of quietly scoring as a match. If only one side is blank, the diff shows it. A page that is + sparse on purpose can carry `data-renderdiff-rendered` on an element its own content renders. - **Blocks analytics, consent and A/B tooling** at the network layer on both sides — GTM, GA, Segment, Intercom, Hotjar, LinkedIn, Meta, OneTrust, CookieLaw, VWO. Blocked rather than hidden, because A/B tools rewrite arbitrary DOM and CSS can't pre-empt an diff --git a/docs/scoping.md b/docs/scoping.md index 723bcd5..c30a43a 100644 --- a/docs/scoping.md +++ b/docs/scoping.md @@ -125,3 +125,8 @@ An element marked `data-renderdiff-mask` is masked like a config mask, on every both sides. The attribute's value is its reason in the report, `rotating quote (data-renderdiff-mask)` for `data-renderdiff-mask="rotating quote"`, and in the result's `masked` in `results.json`. + +A page under 200 characters of text and no visible media on both sides is refused as `page did +not render`. If it is sparse on purpose, like a fixture page, mark an element its own content +renders `data-renderdiff-rendered`. Not the layout shell (``, ``, a layout around the +error boundary): it is there when the page never mounts too, and that page would pass. diff --git a/skills/renderdiff/reference.md b/skills/renderdiff/reference.md index 6f03f7c..a6ed786 100644 --- a/skills/renderdiff/reference.md +++ b/skills/renderdiff/reference.md @@ -123,7 +123,7 @@ Report (`--open`, for a human): green from 99.5%, amber to 97%, red below. | --- | --- | | `HTTP 404 at …` (any ≥ 400) | That side answered an error: a route missing on one origin, or a server not ready. | | `not an HTML page: server sent … at …` | The route is a file (PDF, image, JSON, download), not a page. Drop it from the routes. Never retried. | -| `page did not render on either side (…)` | Both sides under 200 chars of text and no visible image, SVG, canvas, video, iframe or form control. Usually a failed asset or an app that never mounted. One blank side isn't refused; it scores low. | +| `page did not render on either side (…)` | Both sides under 200 chars of text and no visible image, SVG, canvas, video, iframe or form control. Usually a failed asset or an app that never mounted. One blank side isn't refused; it scores low. A page that is sparse on purpose (a fixture page) passes with `data-renderdiff-rendered` on a visible element (32×32px or more) its own content renders, never the layout shell (``, ``, a layout around the error boundary): there it vouches for a page that never mounted too. For a page you can't edit, set it in `beforeShot` only after checking the expected content is there. | | `page did not settle: Npx unstable at …` | Over 25% of the page kept changing, on two loads. `..unsettled-a/-b.png` are the two differing frames, and the message gives the box they differ in. Mask or stop the moving part. | | `page is Npx tall at Wpx wide, past the …px sanity ceiling …` | Taller than 57,600,000px ÷ width (never under 40,000px): a runaway layout, not a page. A long page still compares one element with `--selector` or `@sel` (`element is …` when that element itself is past it). | | `selector "…" matched N elements on left (/a), expected 1` | Make it unique (an id, `data-test`, `>> nth=0`), or list the route once per element. | @@ -201,6 +201,7 @@ aria: identical on 2 of 3 route(s), differs /pricing | Tool | Legitimate for | Never for | | --- | --- | --- | | Mask (`--mask `, config `masks` with `reason`, `--mask-media`, `data-renderdiff-mask=""` in the page) | Content *proven* to vary per load: a base-vs-base baseline differs there (clocks, star counts, rotating quotes, embeds). | Anything that differs only after your change. | +| `data-renderdiff-rendered` in the page | A page sparse on purpose, refused as `page did not render`; on an element its own content renders. | The layout shell, or getting past a page that really didn't render. | | `--accept ""` | A difference you opened that exists before your change (baseline vs production) or is intended, with a reason saying which. | Getting green. | | `--threshold <0..1>` | A CI gate on the exit code. | Reaching green; the difference is still there. | diff --git a/src/capture.ts b/src/capture.ts index 4d3fe7b..b2a38c1 100644 --- a/src/capture.ts +++ b/src/capture.ts @@ -387,7 +387,9 @@ export async function capture( ((root === document.documentElement ? body : root) as HTMLElement).innerText || "", pageText: body.innerText?.length ?? 0, media: Array.from( - document.querySelectorAll("img, svg, canvas, video, iframe, input, textarea, select"), + document.querySelectorAll( + "img, svg, canvas, video, iframe, input, textarea, select, [data-renderdiff-rendered]", + ), ).some((e) => { const r = e.getBoundingClientRect(); return r.width * r.height >= 1024; diff --git a/src/probe.ts b/src/probe.ts index 0d519e6..eb64276 100644 --- a/src/probe.ts +++ b/src/probe.ts @@ -3,7 +3,7 @@ import { prepare } from "./capture.ts"; import { collapseInherited, pageError, SHOWN, type Snapshot } from "./computed.ts"; import { CONTEXT_OPTIONS, masksFor, type ResolvedConfig, type Route } from "./config.ts"; import { SNAP_PROPS, snapshot } from "./locate.ts"; -import { COMPUTED_TIMEOUT, launch, shoot } from "./run.ts"; +import { BLANK_HINT, COMPUTED_TIMEOUT, launch, shoot } from "./run.ts"; // ponytail: x/y also list the siblings a grown box pushes down; drop a y a previous sibling's height explains if it gets noisy const FIELDS = [...SNAP_PROPS, "x", "y"]; @@ -31,7 +31,7 @@ async function load( const pages = { left: await ctx.newPage() } as Record<"left" | "right", Page>; await prepare(pages.left, cfg, now); const shot = await shoot(pages, url, route, "left", width, cfg, now); - if (shot.blank) throw new Error(`page did not render (${shot.blank})`); + if (shot.blank) throw new Error(`page did not render (${shot.blank}): ${BLANK_HINT}`); const page = pages.left; const snap: Snapshot = await snapshot(page, masksFor(cfg, page, route), COMPUTED_TIMEOUT, null); const at = await page.evaluate( diff --git a/src/run.ts b/src/run.ts index 76703b2..391cb4b 100644 --- a/src/run.ts +++ b/src/run.ts @@ -54,6 +54,10 @@ const SHOT_TIMEOUT = (Math.max(0, Number(process.env.RENDERDIFF_SHOT_TIMEOUT)) | /** Past this share of a page unstable, too little was measured for a score to mean anything. */ const MAX_UNSTABLE_SHARE = 0.25; +export const BLANK_HINT = + "under 200 chars of text and no visible image, SVG, canvas, video, iframe or form control; " + + "if the page is meant to be this sparse, add data-renderdiff-rendered to a visible element its own " + + "content renders (not the layout shell, or a page that never mounted passes too)"; interface Job { route: Route; @@ -365,7 +369,7 @@ export async function run({ if (leftShot.blank && rightShot.blank) { throw new Error( `page did not render on either side ` + - `(left: ${leftShot.blank}; right: ${rightShot.blank}) at ${key}`, + `(left: ${leftShot.blank}; right: ${rightShot.blank}) at ${key}: ${BLANK_HINT}`, ); } // A replay that went live in part compared against two different moments of the reference. diff --git a/tests/visual/guardrails.test.ts b/tests/visual/guardrails.test.ts index 5403209..4b95991 100644 --- a/tests/visual/guardrails.test.ts +++ b/tests/visual/guardrails.test.ts @@ -108,6 +108,7 @@ describe("the render gate", () => { assert.match(report.errors[0]!.message, /page did not render/); // The message carries the numbers that explain the refusal. Measured: text=17 chars. assert.match(report.errors[0]!.message, /text=\d+ chars/); + assert.match(report.errors[0]!.message, /under 200 chars .+ add data-renderdiff-rendered/); }); test("results.json lists the refused route as an error, beside the compared one", () => { @@ -170,6 +171,17 @@ describe("the render gate", () => { assert.match(report.errors[0]!.message, /page did not render/); }); + test("a sparse page that marks itself rendered is compared, not refused", async () => { + const file = path.join(tempDir(), "rendered.mjs"); + fs.writeFileSync( + file, + 'export default { beforeShot: async (page) => { await page.evaluate(() => document.querySelector("p").setAttribute("data-renderdiff-rendered", "")); } };\n', + ); + const run = await capture(["/blank"], { config: file }); + assert.equal(run.code, 0, context(run)); + assert.equal(cardFor(run, "/blank").diffPixels, 0, context(run)); + }); + test("a short, nearly textless page with form controls is compared, not refused", async () => { // A sign-in form: under 200 chars of text, which the gate used to refuse on its own. const run = await capture(["/short"], { widths: [1440, 375] }); @@ -1229,7 +1241,7 @@ describe("the run summary", () => { // In full: the reason is what tells a blank page from a broken asset pipeline. assert.match( blank.errors[0]!.message, - /^page did not render on either side \(left: .+; right: .+\) at \/blank$/, + /^page did not render on either side \(left: .+; right: .+\) at \/blank: under 200 chars .+data-renderdiff-rendered.+$/, ); assert.deepEqual([verdict.differ, verdict.routes, verdict.errors], [0, 2, 1]); }); From ee6119c894f28a3f4c95a1064d5075884b73405a Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:07:15 +0100 Subject: [PATCH 2/2] Fix marker size wording; space BLANK_HINT Co-Authored-By: Claude Opus 5.5 --- skills/renderdiff/reference.md | 2 +- src/run.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/skills/renderdiff/reference.md b/skills/renderdiff/reference.md index a6ed786..5c92f9f 100644 --- a/skills/renderdiff/reference.md +++ b/skills/renderdiff/reference.md @@ -123,7 +123,7 @@ Report (`--open`, for a human): green from 99.5%, amber to 97%, red below. | --- | --- | | `HTTP 404 at …` (any ≥ 400) | That side answered an error: a route missing on one origin, or a server not ready. | | `not an HTML page: server sent … at …` | The route is a file (PDF, image, JSON, download), not a page. Drop it from the routes. Never retried. | -| `page did not render on either side (…)` | Both sides under 200 chars of text and no visible image, SVG, canvas, video, iframe or form control. Usually a failed asset or an app that never mounted. One blank side isn't refused; it scores low. A page that is sparse on purpose (a fixture page) passes with `data-renderdiff-rendered` on a visible element (32×32px or more) its own content renders, never the layout shell (``, ``, a layout around the error boundary): there it vouches for a page that never mounted too. For a page you can't edit, set it in `beforeShot` only after checking the expected content is there. | +| `page did not render on either side (…)` | Both sides under 200 chars of text and no visible image, SVG, canvas, video, iframe or form control. Usually a failed asset or an app that never mounted. One blank side isn't refused; it scores low. A page that is sparse on purpose (a fixture page) passes with `data-renderdiff-rendered` on a visible element (at least 1024px² in area) its own content renders, never the layout shell (``, ``, a layout around the error boundary): there it vouches for a page that never mounted too. For a page you can't edit, set it in `beforeShot` only after checking the expected content is there. | | `page did not settle: Npx unstable at …` | Over 25% of the page kept changing, on two loads. `..unsettled-a/-b.png` are the two differing frames, and the message gives the box they differ in. Mask or stop the moving part. | | `page is Npx tall at Wpx wide, past the …px sanity ceiling …` | Taller than 57,600,000px ÷ width (never under 40,000px): a runaway layout, not a page. A long page still compares one element with `--selector` or `@sel` (`element is …` when that element itself is past it). | | `selector "…" matched N elements on left (/a), expected 1` | Make it unique (an id, `data-test`, `>> nth=0`), or list the route once per element. | diff --git a/src/run.ts b/src/run.ts index 391cb4b..1430d60 100644 --- a/src/run.ts +++ b/src/run.ts @@ -54,6 +54,7 @@ const SHOT_TIMEOUT = (Math.max(0, Number(process.env.RENDERDIFF_SHOT_TIMEOUT)) | /** Past this share of a page unstable, too little was measured for a score to mean anything. */ const MAX_UNSTABLE_SHARE = 0.25; + export const BLANK_HINT = "under 200 chars of text and no visible image, SVG, canvas, video, iframe or form control; " + "if the page is meant to be this sparse, add data-renderdiff-rendered to a visible element its own " +