From 2247d5151b36d2051efc48a82eda0062755edcc2 Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:44:25 +0100 Subject: [PATCH 1/2] Add renderdiff probe: what varies between loads of one URL Loads one URL --loads times (default 8) in fresh contexts through a run's own capture, then diffs every element's computed style and position in its parent across the loads, collapsing what a parent's change explains. Exits 0 stable, 1 varies, 2 bad arguments, 3 a load errored. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 ++ skills/renderdiff/SKILL.md | 3 +- skills/renderdiff/reference.md | 10 +++ src/cli.ts | 68 +++++++++++++- src/computed.ts | 5 +- src/probe.ts | 123 ++++++++++++++++++++++++++ src/run.ts | 6 +- tests/fixtures/pages/README.md | 2 +- tests/fixtures/pages/bistable/page.ts | 45 ++++++++++ tests/helpers/corpus.ts | 3 +- tests/unit/cli-args.test.ts | 20 +++++ tests/visual/guardrails.test.ts | 18 ++++ 12 files changed, 301 insertions(+), 8 deletions(-) create mode 100644 src/probe.ts create mode 100644 tests/fixtures/pages/bistable/page.ts diff --git a/README.md b/README.md index 255317f..8fd1e18 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,12 @@ out dir with the shots a partial run also took (say, a few routes rerun under a replaced by its, and its other shots added. Each merged shot carries `from`: [docs/output.md](https://github.com/tmashara/renderdiff/blob/main/docs/output.md#merge). +`renderdiff probe [--width ] [--loads ]` loads one URL 8 times (`--loads`), each +in a fresh browser context as a run loads a side, and prints each element whose computed +style or position in its parent varies between loads, with its values and how often each +showed: what to mask when base against base differs with no `varies` line and nothing +unstable. Exits 0 if nothing varies, 1 if something does, 3 if a load errored. + ## Why a clean result is trustworthy - **Refuses to start** unless both origins answer: two error pages would score ~100%. diff --git a/skills/renderdiff/SKILL.md b/skills/renderdiff/SKILL.md index 534747e..1bd3625 100644 --- a/skills/renderdiff/SKILL.md +++ b/skills/renderdiff/SKILL.md @@ -38,7 +38,8 @@ Pixels decide the score and the exit code in every mode. differs varies per load: mask it (a `varies` line prints the mask; `--mask-media` for iframes, `--mask ` or a config mask with a `reason`; video is already frozen; a looping carousel: stop it with a `before` hook on both sides, or mask it), and rerun - until clean. Against production instead: open each difference, then + until clean. A difference with no `varies` line and nothing unstable: + `renderdiff probe --width ` names what varies between loads; mask that. Against production instead: open each difference, then `--accept "differs before the change"`. § Masks, accepts and thresholds. 3. **Change, run all routes at default widths, read each row in this order**: `ERROR` → `failed` (a missing stylesheet or font explains the whole row: fix, rerun) → `text` → diff --git a/skills/renderdiff/reference.md b/skills/renderdiff/reference.md index 5895802..c7d35ae 100644 --- a/skills/renderdiff/reference.md +++ b/skills/renderdiff/reference.md @@ -317,6 +317,16 @@ widths are unioned in `meta`; accepted flags stay as each run set them. `evidenc Not verified. Exits 2 on different origins, an `--out` that is or is inside an input or holds foreign files, or two shots writing one file. A merged out dir refuses `--rerun-failed`. +## `probe` + +`renderdiff probe [--width ] [--loads ]` loads one URL `--loads` times (default +8, in parallel up to `--concurrency`), each in a fresh context with a cold cache, through the +same capture as a run's left side (config masks, hooks, `--left-storage-state`). One line per +varying element and property, ` ×, ...`, most frequent first; `x` +and `y` are its position in its parent, `present` means some loads lack it. First 20 shown. +One width (the first default without `--width`), no `--selector`, no `--out`. Exits 0 stable, +1 varies, 2 bad arguments or the origin down, 3 a load errored. + ## results.json For scripts and drilling into one route; the loop doesn't need it. One result per line, so diff --git a/src/cli.ts b/src/cli.ts index 4c8e5b8..b0bf734 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -182,6 +182,7 @@ const HELP = ` renderdiff http://localhost:3000 https://example.com /pricing::/plans renderdiff report [--verdict ] [--title ] renderdiff merge --out + renderdiff probe [--width ] [--loads ] An argument starting with http:// or https:// is an origin: the first is left (your change), the second right (the reference); or pass --left / --right. A route is a @@ -208,6 +209,11 @@ const HELP = ` renderdiff merge writes 's results to --out with each shot also has replaced by the partial run's, and its other shots added. Both must compare the same origins. + renderdiff probe loads one URL --loads times (default 8, in parallel up to --concurrency), + each in a fresh browser context as a run loads a side, and lists the elements whose computed + style or position in their parent varies between loads: what to mask. Exits 0 if nothing + varies, 1 if something does, 3 if a load errored. + Writes report.html, results.json and crops of each change to --out. Exits 1 if any shot differs and isn't accepted (with --threshold, only if one scores below it; --threshold 0 never fails on a diff), 3 if a route errored (unmeasured) or the run @@ -414,12 +420,71 @@ async function merge(argv: string[]): Promise { } } +/** Takes `--loads ` out of `argv`, for `renderdiff probe`. */ +function takeLoads(argv: string[]): number { + let loads = 8; + for (let i = 0; i < argv.length; i++) { + const raw = argv[i]!; + const eq = raw.indexOf("="); + const v = eq === -1 ? raw : raw.slice(0, eq); + if (v === "--loads") { + const value = eq === -1 ? argv[i + 1] : raw.slice(eq + 1); + if (value === undefined || value === "" || value.startsWith("-")) { + throw new Error("--loads needs a value"); + } + loads = num("--loads", value); + if (!Number.isInteger(loads) || loads < 2) { + throw new Error(`--loads expects an integer of at least 2, got ${loads}`); + } + argv.splice(i--, eq === -1 ? 2 : 1); + } + } + return loads; +} + +/** `renderdiff probe `: load one URL several times and list what varies between loads. */ +async function probe(a: Args, loads: number): Promise { + if (!a.left) die("probe needs a URL: renderdiff probe "); + if (a.right || a.routes.length) die("probe takes one URL, with its path"); + if ((a.widths?.length ?? 0) > 1) die("probe takes one --width"); + if (a.selector) die("probe does not take --selector; it walks the whole page"); + if (!URL.canParse(a.left)) die(`not a URL: ${a.left}`); + const u = new URL(a.left); + let route: Route; + try { + cfg = resolveConfig(fileConfig, { ...a, left: u.origin, right: u.origin }); + [route] = parseRoutes( + [u.pathname + u.search], + undefined, + fileConfig.before, + fileConfig.beforeShot, + ) as [Route]; + } catch (e) { + die(e instanceof Error ? e.message : String(e)); + } + if (configPath) console.log(`Config: ${configPath}`); + await preflight(cfg.left, cfg.left); + const { probe } = await import("./probe.ts"); + try { + process.exit( + await probe(cfg.left + route.left, route, a.widths?.[0] ?? cfg.widths[0]!, cfg, loads), + ); + } catch (e) { + console.error(`✗ ${e instanceof Error ? e.message : String(e)}`); + process.exit(3); + } +} + if (process.argv[2] === "report") await report(process.argv.slice(3)); if (process.argv[2] === "merge") await merge(process.argv.slice(3)); +const probing = process.argv[2] === "probe"; +let loads = 0; let args: Args; try { - args = parseArgs(process.argv.slice(2)); + const argv = process.argv.slice(probing ? 3 : 2); + if (probing) loads = takeLoads(argv); + args = parseArgs(argv); } catch (e) { die(e instanceof Error ? e.message : String(e)); } @@ -441,6 +506,7 @@ const { config: fileConfig, path: configPath } = await loadConfig(args.config).c ); let cfg: ResolvedConfig; +if (probing) await probe(args, loads); try { cfg = resolveConfig(fileConfig, args); } catch (e) { diff --git a/src/computed.ts b/src/computed.ts index 04cb120..b9c5f2b 100644 --- a/src/computed.ts +++ b/src/computed.ts @@ -90,7 +90,10 @@ export const pageError = (e: unknown) => { * With the snapshots, a box whose own size is set in px on either side is kept, and a * percentage height inside a px-height parent whose height changed is the one that follows. */ -export function collapseInherited(changes: StyleChange[], snaps: Snapshot[] = []): StyleChange[] { +export function collapseInherited( + changes: C[], + snaps: Snapshot[] = [], +): C[] { const raw = new Set(changes.map((c) => `${c.key}\0${c.prop}\0${c.values.join("\0")}`)); const wide = new Set(changes.filter((c) => c.prop === "width").map((c) => c.key)); const tall = new Set(changes.filter((c) => c.prop === "height").map((c) => c.key)); diff --git a/src/probe.ts b/src/probe.ts new file mode 100644 index 0000000..42e410d --- /dev/null +++ b/src/probe.ts @@ -0,0 +1,123 @@ +import type { Page } from "playwright"; +import { prepare } from "./capture.ts"; +import { collapseInherited, pageError, SHOWN, type Snapshot } from "./computed.ts"; +import { CONTEXT_OPTIONS, masksFor, type ResolvedConfig, type Route } from "./config.ts"; +import { PROPS, snapshot } from "./locate.ts"; +import { COMPUTED_TIMEOUT, launch, shoot } from "./run.ts"; + +/** `renderdiff probe`: what varies between loads of one URL, from computed styles and positions. */ + +// ponytail: x/y also list the siblings a grown box pushes down; drop a y a previous sibling's height explains if it gets noisy +const FIELDS = [...PROPS, "content", "x", "y"]; + +interface Change { + key: string; + prop: string; + values: string[]; +} + +/** One load in a fresh context, as a run loads a side: every element's styles, then its position in its parent. */ +async function load( + browser: Awaited>["browser"], + url: string, + route: Route, + width: number, + cfg: ResolvedConfig, + now: number, +): Promise<{ snap: Snapshot; row: Record }> { + const ctx = await browser.newContext({ + ...CONTEXT_OPTIONS, + storageState: cfg.storageState.left ?? undefined, + }); + try { + const pages = { left: await ctx.newPage() } as Record<"left" | "right", Page>; + await prepare(pages.left, cfg, now); + const shot = await shoot(pages, url, route, "left", width, cfg, now); + if (shot.blank) throw new Error(`page did not render (${shot.blank})`); + const page = pages.left; + const snap: Snapshot = await snapshot(page, masksFor(cfg, page, route), COMPUTED_TIMEOUT, null); + const at = await page.evaluate( + (keys) => + Object.fromEntries( + keys.flatMap((k) => { + const e = document.querySelector(k); + if (!e?.parentElement) return []; + const [r, p] = [e, e.parentElement].map((n) => n.getBoundingClientRect()); + return [[k, [String(Math.round(r!.x - p!.x)), String(Math.round(r!.y - p!.y))]]]; + }), + ), + Object.keys(snap).filter((k) => !k.includes("::")), + ); + const row = Object.fromEntries( + Object.entries(snap).map(([k, s]) => [k, [...s.styles, ...(at[k] ?? ["", ""])]]), + ); + return { snap, row }; + } finally { + await ctx.close().catch(() => {}); + } +} + +export async function probe( + url: string, + route: Route, + width: number, + cfg: ResolvedConfig, + loads: number, +): Promise { + const { browser } = await launch(); + const now = Math.floor(Date.now() / 60_000) * 60_000; + const done: { snap: Snapshot; row: Record }[] = []; + let next = 0; + const worker = async () => { + for (let k = next++; k < loads; k = next++) { + try { + done[k] = await load(browser, url, route, width, cfg, now); + } catch (e) { + throw new Error(`load ${k + 1}: ${pageError(e)}`); + } + } + }; + try { + await Promise.all(Array.from({ length: Math.min(cfg.concurrency, loads) }, worker)); + } finally { + await browser.close(); + } + + const rows = done.map((d) => d.row); + const keys = [...new Set(rows.flatMap((r) => Object.keys(r)))]; + const absent: string[] = []; + const changes: Change[] = []; + for (const key of keys) { + const vs = rows.map((r) => r[key]); + if (vs.some((v) => !v)) { + if (!absent.some((a) => key.startsWith(a + " > ") || key.startsWith(a + "::"))) { + absent.push(key); + changes.push({ key, prop: "present", values: vs.map((v) => (v ? "yes" : "no")) }); + } + continue; + } + FIELDS.forEach((prop, i) => { + const values = vs.map((v) => v![i]!); + if (new Set(values).size > 1) changes.push({ key, prop, values }); + }); + } + const varying = collapseInherited( + changes, + done.map((d) => d.snap), + ); + + console.log(`Probe: ${url} @${width}, ${loads} loads`); + if (!varying.length) { + console.log(`stable over ${loads} loads (${keys.length} elements)`); + return 0; + } + console.log(`${varying.length} varying:`); + for (const c of varying.slice(0, SHOWN)) { + const tally = new Map(); + for (const v of c.values) tally.set(v, (tally.get(v) ?? 0) + 1); + const counts = [...tally].sort((a, b) => b[1] - a[1]).map(([v, n]) => `${v} ×${n}`); + console.log(` ${c.key} ${c.prop} ${counts.join(", ")}`); + } + if (varying.length > SHOWN) console.log(` (+${varying.length - SHOWN} more)`); + return 1; +} diff --git a/src/run.ts b/src/run.ts index 1974ee8..7160faf 100644 --- a/src/run.ts +++ b/src/run.ts @@ -38,7 +38,7 @@ import { /** A busy main thread can stall the probe; the elements are a hint, so give up rather than wait. */ const LOCATE_TIMEOUT = 5_000; /** A full-page walk reads every element's style; past this, report the snapshot as failed. */ -const COMPUTED_TIMEOUT = 15_000; +export const COMPUTED_TIMEOUT = 15_000; /** Bounds one capture: a renderer that dies or never returns from an evaluate would hang the run. */ const SHOT_TIMEOUT = (Math.max(0, Number(process.env.RENDERDIFF_SHOT_TIMEOUT)) || 120) * 1000; @@ -788,7 +788,7 @@ const ARGS = [ * A missing browser binary is the likeliest first-run failure, so fall back to the system * Chrome (both sides share it, so the comparison stays symmetric), then name the fix. */ -async function launch() { +export async function launch() { try { return { browser: await chromium.launch({ args: ARGS }), channel: null }; } catch (e) { @@ -814,7 +814,7 @@ const transient = (e: unknown) => (e instanceof Error && /net::ERR_|Execution context was destroyed/.test(e.message))); /** One retry for a transient failure: a live site can time out once on a cold edge. */ -async function shoot( +export async function shoot( pages: Record<"left" | "right", Page>, url: string, route: Route, diff --git a/tests/fixtures/pages/README.md b/tests/fixtures/pages/README.md index bd9edea..38c3b40 100644 --- a/tests/fixtures/pages/README.md +++ b/tests/fixtures/pages/README.md @@ -19,7 +19,7 @@ Three groups: elements each change names (`margin`, `color`, `nav`), and a lost `::before` line and a re-parented pairing (`divider`). - **Guard rails** — `blank`, `tall`, `animated`, `lazy`, `lazy-deep`, `lazy-js`, `lazy-grow`, `consent`, - `drifting`, `tracker`, `clock`, `ticker`, `unsettled`, `unsettled-once`, `sprite`, `short`, `tall-region`, `reorder`, `stripe`, `frameset`, `menu`, `scroll-spy`, `signed-in`, `fixed`, `inserted`, `labelled`, `broken-asset`, `media`, `long`, `video`, `video-seek`, `hang`, `origin`, `cooperate`, `mask-geometry`. + `drifting`, `tracker`, `clock`, `ticker`, `unsettled`, `unsettled-once`, `sprite`, `short`, `tall-region`, `reorder`, `stripe`, `frameset`, `menu`, `scroll-spy`, `signed-in`, `fixed`, `inserted`, `labelled`, `broken-asset`, `media`, `long`, `video`, `video-seek`, `hang`, `origin`, `cooperate`, `mask-geometry`, `bistable`. One per defence in `src/capture.ts` / `src/config.ts`. ## Authoring rules diff --git a/tests/fixtures/pages/bistable/page.ts b/tests/fixtures/pages/bistable/page.ts new file mode 100644 index 0000000..1cabfde --- /dev/null +++ b/tests/fixtures/pages/bistable/page.ts @@ -0,0 +1,45 @@ +/** + * GUARD RAIL for `renderdiff probe`. + * + * A player box that is 366px tall on odd renders and 380px on even ones, the way an embed + * that sometimes gets its controls bar does. Every load is still, so a run only sees it as a + * base-vs-base difference; probe must name the box and its two heights. Generated because the + * counter has to live on the server. + */ + +let renders = 0; + +const page = () => ` + + + +Layout parity + + + + +
+

Bistable guard rail

+

The box below is one of two heights, alternating per render. Each load is still, but two loads of the same page disagree.

+ +
+ +
+

Static content

+

Everything else on this page is fixed, so the only thing that varies between loads is the player's height and what it pushes down.

+
+ +
+

Notes

+

A difference between two builds of the same base can come from the page itself. Loading one side several times shows which elements vary on their own.

+
+
+ + +`; + +export const left = page; +export const right = page; diff --git a/tests/helpers/corpus.ts b/tests/helpers/corpus.ts index 995cb39..7912273 100644 --- a/tests/helpers/corpus.ts +++ b/tests/helpers/corpus.ts @@ -60,12 +60,13 @@ export const GUARDRAILS = [ "origin", "cooperate", "mask-geometry", + "bistable", ] as const; export const ALL: string[] = [...EQUIVALENCE, ...CONTROLS, ...GUARDRAILS]; /** Routes whose markup comes from a `page.ts` generator instead of static files. */ -export const GENERATED = ["tall", "drifting", "unsettled-once"]; +export const GENERATED = ["tall", "drifting", "unsettled-once", "bistable"]; /** Routes that deliberately do not link `shared/base.css`. */ export const SELF_CONTAINED = ["blank", "sprite", "tall", "frameset"]; diff --git a/tests/unit/cli-args.test.ts b/tests/unit/cli-args.test.ts index 7cdc077..e870e65 100644 --- a/tests/unit/cli-args.test.ts +++ b/tests/unit/cli-args.test.ts @@ -396,6 +396,26 @@ describe("accept and rerun", () => { }); }); +describe("probe", () => { + const U = `${L}/a`; + test("needs one URL", async () => { + await rejects(["probe"], "probe needs a URL"); + await rejects(["probe", U, R], "probe takes one URL"); + await rejects(["probe", U, "/b"], "probe takes one URL"); + }); + + test("--loads is an integer of at least 2", async () => { + await rejects(["probe", U, "--loads", "1"], "--loads expects an integer of at least 2, got 1"); + await rejects(["probe", U, "--loads=2.5"], "--loads expects an integer of at least 2"); + await rejects(["probe", U, "--loads"], "--loads needs a value"); + }); + + test("one width, no selector", async () => { + await rejects(["probe", U, "--width", "1440", "--width", "375"], "probe takes one --width"); + await rejects(["probe", U, "--selector", "main"], "probe does not take --selector"); + }); +}); + describe("preflight", () => { // Port 1 is reserved and never listening, so this is a closed port rather than a race // against something we just shut down. diff --git a/tests/visual/guardrails.test.ts b/tests/visual/guardrails.test.ts index f3805cb..1b4ea29 100644 --- a/tests/visual/guardrails.test.ts +++ b/tests/visual/guardrails.test.ts @@ -1392,3 +1392,21 @@ describe("shot deadline", () => { assert.equal(site.hits("/menu") - before, 2, `the hook was retried${context(run)}`); }); }); + +describe("probe", () => { + test("names the box whose height varies between loads, not its parent", async () => { + const before = site.hits("/bistable"); + const args = ["probe", site.left + "/bistable", "--width", "1440", "--loads", "4"]; + const run = await runCli(args, { noOut: true }); + assert.equal(run.code, 1, context(run)); + assert.ok(site.hits("/bistable") - before >= 4, context(run)); + assert.match(run.stdout, /div\.player height (366|380)px ×\d+, (366|380)px ×\d+/, context(run)); + assert.doesNotMatch(run.stdout, /div\.page height/, context(run)); + }); + + test("a still page is stable", async () => { + const run = await runCli(["probe", site.left + "/", "--loads", "3"], { noOut: true }); + assert.equal(run.code, 0, context(run)); + assert.match(run.stdout, /stable over 3 loads/, context(run)); + }); +}); From b07bd0d2a9fa22f6a36f50338564a660ce97dfbc Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:50:57 +0100 Subject: [PATCH 2/2] Probe: reject --loads in a run test, tidy doc comment and skill wrap Co-Authored-By: Claude Opus 5.5 --- skills/renderdiff/SKILL.md | 3 ++- src/cli.ts | 4 +--- src/probe.ts | 3 +-- tests/unit/cli-args.test.ts | 1 + 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/skills/renderdiff/SKILL.md b/skills/renderdiff/SKILL.md index 1bd3625..e3d2a39 100644 --- a/skills/renderdiff/SKILL.md +++ b/skills/renderdiff/SKILL.md @@ -39,7 +39,8 @@ Pixels decide the score and the exit code in every mode. iframes, `--mask ` or a config mask with a `reason`; video is already frozen; a looping carousel: stop it with a `before` hook on both sides, or mask it), and rerun until clean. A difference with no `varies` line and nothing unstable: - `renderdiff probe --width ` names what varies between loads; mask that. Against production instead: open each difference, then + `renderdiff probe --width ` names what varies between loads; mask that. + Against production instead: open each difference, then `--accept "differs before the change"`. § Masks, accepts and thresholds. 3. **Change, run all routes at default widths, read each row in this order**: `ERROR` → `failed` (a missing stylesheet or font explains the whole row: fix, rerun) → `text` → diff --git a/src/cli.ts b/src/cli.ts index b0bf734..a4aa756 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -466,9 +466,7 @@ async function probe(a: Args, loads: number): Promise { await preflight(cfg.left, cfg.left); const { probe } = await import("./probe.ts"); try { - process.exit( - await probe(cfg.left + route.left, route, a.widths?.[0] ?? cfg.widths[0]!, cfg, loads), - ); + process.exit(await probe(cfg.left + route.left, route, cfg.widths[0]!, cfg, loads)); } catch (e) { console.error(`✗ ${e instanceof Error ? e.message : String(e)}`); process.exit(3); diff --git a/src/probe.ts b/src/probe.ts index 42e410d..f128b96 100644 --- a/src/probe.ts +++ b/src/probe.ts @@ -5,8 +5,6 @@ import { CONTEXT_OPTIONS, masksFor, type ResolvedConfig, type Route } from "./co import { PROPS, snapshot } from "./locate.ts"; import { COMPUTED_TIMEOUT, launch, shoot } from "./run.ts"; -/** `renderdiff probe`: what varies between loads of one URL, from computed styles and positions. */ - // ponytail: x/y also list the siblings a grown box pushes down; drop a y a previous sibling's height explains if it gets noisy const FIELDS = [...PROPS, "content", "x", "y"]; @@ -57,6 +55,7 @@ async function load( } } +/** `renderdiff probe`: what varies between loads of one URL, from computed styles and positions. */ export async function probe( url: string, route: Route, diff --git a/tests/unit/cli-args.test.ts b/tests/unit/cli-args.test.ts index e870e65..52505e7 100644 --- a/tests/unit/cli-args.test.ts +++ b/tests/unit/cli-args.test.ts @@ -408,6 +408,7 @@ describe("probe", () => { await rejects(["probe", U, "--loads", "1"], "--loads expects an integer of at least 2, got 1"); await rejects(["probe", U, "--loads=2.5"], "--loads expects an integer of at least 2"); await rejects(["probe", U, "--loads"], "--loads needs a value"); + await rejects([L, R, "/a", "--loads", "3"], "unknown flag: --loads"); }); test("one width, no selector", async () => {