Add renderdiff probe: what varies between loads of one URL

README.md

@@ -242,6 +242,12 @@ out dir with the shots a partial run also took (say, a few routes rerun under a
 replaced by its, and its other shots added. Each merged shot carries `from`:
 [docs/output.md](https://github.com/tmashara/renderdiff/blob/main/docs/output.md#merge).
 
+`renderdiff probe <url> [--width <n>] [--loads <n>]` loads one URL 8 times (`--loads`), each
+in a fresh browser context as a run loads a side, and prints each element whose computed
+style or position in its parent varies between loads, with its values and how often each
+showed: what to mask when base against base differs with no `varies` line and nothing
+unstable. Exits 0 if nothing varies, 1 if something does, 3 if a load errored.
+
 ## Why a clean result is trustworthy
 
 - **Refuses to start** unless both origins answer: two error pages would score ~100%.

skills/renderdiff/SKILL.md

@@ -38,7 +38,9 @@ Pixels decide the score and the exit code in every mode.
    differs varies per load: mask it (a `varies` line prints the mask; `--mask-media` for
    iframes, `--mask <sel>` or a config mask with a `reason`; video is already frozen; a
    looping carousel: stop it with a `before` hook on both sides, or mask it), and rerun
-   until clean. Against production instead: open each difference, then
+   until clean. A difference with no `varies` line and nothing unstable:
+   `renderdiff probe <right><route> --width <w>` names what varies between loads; mask that.
+   Against production instead: open each difference, then
    `--accept "differs before the change"`. § Masks, accepts and thresholds.
 3. **Change, run all routes at default widths, read each row in this order**: `ERROR` →
    `failed` (a missing stylesheet or font explains the whole row: fix, rerun) → `text` →

skills/renderdiff/reference.md

@@ -318,6 +318,16 @@ widths are unioned in `meta`; accepted flags stay as each run set them. `evidenc
 Not verified. Exits 2 on different origins, an `--out` that is or is inside an input or holds foreign
 files, or two shots writing one file. A merged out dir refuses `--rerun-failed`.
 
+## `probe`
+
+`renderdiff probe <url> [--width <n>] [--loads <n>]` loads one URL `--loads` times (default
+8, in parallel up to `--concurrency`), each in a fresh context with a cold cache, through the
+same capture as a run's left side (config masks, hooks, `--left-storage-state`). One line per
+varying element and property, `<key> <prop> <value> ×<loads>, ...`, most frequent first; `x`
+and `y` are its position in its parent, `present` means some loads lack it. First 20 shown.
+One width (the first default without `--width`), no `--selector`, no `--out`. Exits 0 stable,
+1 varies, 2 bad arguments or the origin down, 3 a load errored.
+
 ## results.json
 
 For scripts and drilling into one route; the loop doesn't need it. One result per line, so

src/cli.ts

@@ -182,6 +182,7 @@ const HELP = `
     renderdiff http://localhost:3000 https://example.com /pricing::/plans
     renderdiff report <out-dir> [--verdict <file>] [--title <text>]
     renderdiff merge <full-out> <partial-out> --out <dir>
+    renderdiff probe <url> [--width <n>] [--loads <n>]
 
   An argument starting with http:// or https:// is an origin: the first is left (your
   change), the second right (the reference); or pass --left / --right. A route is a
@@ -208,6 +209,11 @@ const HELP = `
   renderdiff merge writes <full-out>'s results to --out with each shot <partial-out> also
   has replaced by the partial run's, and its other shots added. Both must compare the same origins.
 
+  renderdiff probe loads one URL --loads times (default 8, in parallel up to --concurrency),
+  each in a fresh browser context as a run loads a side, and lists the elements whose computed
+  style or position in their parent varies between loads: what to mask. Exits 0 if nothing
+  varies, 1 if something does, 3 if a load errored.
+
   Writes report.html, results.json and crops of each change to --out. Exits 1 if any
   shot differs and isn't accepted (with --threshold, only if one scores below it;
   --threshold 0 never fails on a diff), 3 if a route errored (unmeasured) or the run
@@ -414,12 +420,69 @@ async function merge(argv: string[]): Promise<never> {
   }
 }
 
+/** Takes `--loads <n>` out of `argv`, for `renderdiff probe`. */
+function takeLoads(argv: string[]): number {
+  let loads = 8;
+  for (let i = 0; i < argv.length; i++) {
+    const raw = argv[i]!;
+    const eq = raw.indexOf("=");
+    const v = eq === -1 ? raw : raw.slice(0, eq);
+    if (v === "--loads") {
+      const value = eq === -1 ? argv[i + 1] : raw.slice(eq + 1);
+      if (value === undefined || value === "" || value.startsWith("-")) {
+        throw new Error("--loads needs a value");
+      }
+      loads = num("--loads", value);
+      if (!Number.isInteger(loads) || loads < 2) {
+        throw new Error(`--loads expects an integer of at least 2, got ${loads}`);
+      }
+      argv.splice(i--, eq === -1 ? 2 : 1);
+    }
+  }
+  return loads;
+}
+
+/** `renderdiff probe <url>`: load one URL several times and list what varies between loads. */
+async function probe(a: Args, loads: number): Promise<never> {
+  if (!a.left) die("probe needs a URL: renderdiff probe <url>");
+  if (a.right || a.routes.length) die("probe takes one URL, with its path");
+  if ((a.widths?.length ?? 0) > 1) die("probe takes one --width");
+  if (a.selector) die("probe does not take --selector; it walks the whole page");
+  if (!URL.canParse(a.left)) die(`not a URL: ${a.left}`);
+  const u = new URL(a.left);
+  let route: Route;
+  try {
+    cfg = resolveConfig(fileConfig, { ...a, left: u.origin, right: u.origin });
+    [route] = parseRoutes(
+      [u.pathname + u.search],
+      undefined,
+      fileConfig.before,
+      fileConfig.beforeShot,
+    ) as [Route];
+  } catch (e) {
+    die(e instanceof Error ? e.message : String(e));
+  }
+  if (configPath) console.log(`Config: ${configPath}`);
+  await preflight(cfg.left, cfg.left);
+  const { probe } = await import("./probe.ts");
+  try {
+    process.exit(await probe(cfg.left + route.left, route, cfg.widths[0]!, cfg, loads));
+  } catch (e) {
+    console.error(`✗ ${e instanceof Error ? e.message : String(e)}`);
+    process.exit(3);
+  }
+}
+
 if (process.argv[2] === "report") await report(process.argv.slice(3));
 if (process.argv[2] === "merge") await merge(process.argv.slice(3));
 
+const probing = process.argv[2] === "probe";
+let loads = 0;
 let args: Args;
 try {
-  args = parseArgs(process.argv.slice(2));
+  const argv = process.argv.slice(probing ? 3 : 2);
+  if (probing) loads = takeLoads(argv);
+  args = parseArgs(argv);
 } catch (e) {
   die(e instanceof Error ? e.message : String(e));
 }
@@ -441,6 +504,7 @@ const { config: fileConfig, path: configPath } = await loadConfig(args.config).c
 );
 
 let cfg: ResolvedConfig;
+if (probing) await probe(args, loads);
 try {
   cfg = resolveConfig(fileConfig, args);
 } catch (e) {

src/computed.ts

@@ -90,7 +90,10 @@ export const pageError = (e: unknown) => {
  * With the snapshots, a box whose own size is set in px on either side is kept, and a
  * percentage height inside a px-height parent whose height changed is the one that follows.
  */
-export function collapseInherited(changes: StyleChange[], snaps: Snapshot[] = []): StyleChange[] {
+export function collapseInherited<C extends { key: string; prop: string; values: string[] }>(
+  changes: C[],
+  snaps: Snapshot[] = [],
+): C[] {
   const raw = new Set(changes.map((c) => `${c.key}\0${c.prop}\0${c.values.join("\0")}`));
   const wide = new Set(changes.filter((c) => c.prop === "width").map((c) => c.key));
   const tall = new Set(changes.filter((c) => c.prop === "height").map((c) => c.key));

src/probe.ts

@@ -0,0 +1,122 @@
+import type { Page } from "playwright";
+import { prepare } from "./capture.ts";
+import { collapseInherited, pageError, SHOWN, type Snapshot } from "./computed.ts";
+import { CONTEXT_OPTIONS, masksFor, type ResolvedConfig, type Route } from "./config.ts";
+import { PROPS, snapshot } from "./locate.ts";
+import { COMPUTED_TIMEOUT, launch, shoot } from "./run.ts";
+
+// ponytail: x/y also list the siblings a grown box pushes down; drop a y a previous sibling's height explains if it gets noisy
+const FIELDS = [...PROPS, "content", "x", "y"];
+
+interface Change {
+  key: string;
+  prop: string;
+  values: string[];
+}
+
+/** One load in a fresh context, as a run loads a side: every element's styles, then its position in its parent. */
+async function load(
+  browser: Awaited<ReturnType<typeof launch>>["browser"],
+  url: string,
+  route: Route,
+  width: number,
+  cfg: ResolvedConfig,
+  now: number,
+): Promise<{ snap: Snapshot; row: Record<string, string[]> }> {
+  const ctx = await browser.newContext({
+    ...CONTEXT_OPTIONS,
+    storageState: cfg.storageState.left ?? undefined,
+  });
+  try {
+    const pages = { left: await ctx.newPage() } as Record<"left" | "right", Page>;
+    await prepare(pages.left, cfg, now);
+    const shot = await shoot(pages, url, route, "left", width, cfg, now);
+    if (shot.blank) throw new Error(`page did not render (${shot.blank})`);
+    const page = pages.left;
+    const snap: Snapshot = await snapshot(page, masksFor(cfg, page, route), COMPUTED_TIMEOUT, null);
+    const at = await page.evaluate(
+      (keys) =>
+        Object.fromEntries(
+          keys.flatMap((k) => {
+            const e = document.querySelector(k);
+            if (!e?.parentElement) return [];
+            const [r, p] = [e, e.parentElement].map((n) => n.getBoundingClientRect());
+            return [[k, [String(Math.round(r!.x - p!.x)), String(Math.round(r!.y - p!.y))]]];
+          }),
+        ),
+      Object.keys(snap).filter((k) => !k.includes("::")),
+    );
+    const row = Object.fromEntries(
+      Object.entries(snap).map(([k, s]) => [k, [...s.styles, ...(at[k] ?? ["", ""])]]),
+    );
+    return { snap, row };
+  } finally {
+    await ctx.close().catch(() => {});
+  }
+}
+
+/** `renderdiff probe`: what varies between loads of one URL, from computed styles and positions. */
+export async function probe(
+  url: string,
+  route: Route,
+  width: number,
+  cfg: ResolvedConfig,
+  loads: number,
+): Promise<number> {
+  const { browser } = await launch();
+  const now = Math.floor(Date.now() / 60_000) * 60_000;
+  const done: { snap: Snapshot; row: Record<string, string[]> }[] = [];
+  let next = 0;
+  const worker = async () => {
+    for (let k = next++; k < loads; k = next++) {
+      try {
+        done[k] = await load(browser, url, route, width, cfg, now);
+      } catch (e) {
+        throw new Error(`load ${k + 1}: ${pageError(e)}`);
+      }
+    }
+  };
+  try {
+    await Promise.all(Array.from({ length: Math.min(cfg.concurrency, loads) }, worker));
+  } finally {
+    await browser.close();
+  }
+
+  const rows = done.map((d) => d.row);
+  const keys = [...new Set(rows.flatMap((r) => Object.keys(r)))];
+  const absent: string[] = [];
+  const changes: Change[] = [];
+  for (const key of keys) {
+    const vs = rows.map((r) => r[key]);
+    if (vs.some((v) => !v)) {
+      if (!absent.some((a) => key.startsWith(a + " > ") || key.startsWith(a + "::"))) {
+        absent.push(key);
+        changes.push({ key, prop: "present", values: vs.map((v) => (v ? "yes" : "no")) });
+      }
+      continue;
+    }
+    FIELDS.forEach((prop, i) => {
+      const values = vs.map((v) => v![i]!);
+      if (new Set(values).size > 1) changes.push({ key, prop, values });
+    });
+  }
+  const varying = collapseInherited(
+    changes,
+    done.map((d) => d.snap),
+  );
+
+  console.log(`Probe: ${url} @${width}, ${loads} loads`);
+  if (!varying.length) {
+    console.log(`stable over ${loads} loads (${keys.length} elements)`);
+    return 0;
+  }
+  console.log(`${varying.length} varying:`);
+  for (const c of varying.slice(0, SHOWN)) {
+    const tally = new Map<string, number>();
+    for (const v of c.values) tally.set(v, (tally.get(v) ?? 0) + 1);
+    const counts = [...tally].sort((a, b) => b[1] - a[1]).map(([v, n]) => `${v} ×${n}`);
+    console.log(`  ${c.key} ${c.prop} ${counts.join(", ")}`);
+  }
+  if (varying.length > SHOWN) console.log(`  (+${varying.length - SHOWN} more)`);
+  return 1;
+}

src/run.ts

@@ -38,7 +38,7 @@ import {
 /** A busy main thread can stall the probe; the elements are a hint, so give up rather than wait. */
 const LOCATE_TIMEOUT = 5_000;
 /** A full-page walk reads every element's style; past this, report the snapshot as failed. */
-const COMPUTED_TIMEOUT = 15_000;
+export const COMPUTED_TIMEOUT = 15_000;
 /** Bounds one capture: a renderer that dies or never returns from an evaluate would hang the run. */
 const SHOT_TIMEOUT = (Math.max(0, Number(process.env.RENDERDIFF_SHOT_TIMEOUT)) || 120) * 1000;
 
@@ -789,7 +789,7 @@ const ARGS = [
  * A missing browser binary is the likeliest first-run failure, so fall back to the system
  * Chrome (both sides share it, so the comparison stays symmetric), then name the fix.
  */
-async function launch() {
+export async function launch() {
   try {
     return { browser: await chromium.launch({ args: ARGS }), channel: null };
   } catch (e) {
@@ -815,7 +815,7 @@ const transient = (e: unknown) =>
     (e instanceof Error && /net::ERR_|Execution context was destroyed/.test(e.message)));
 
 /** One retry for a transient failure: a live site can time out once on a cold edge. */
-async function shoot(
+export async function shoot(
   pages: Record<"left" | "right", Page>,
   url: string,
   route: Route,

tests/fixtures/pages/README.md

@@ -19,7 +19,7 @@ Three groups:
   elements each change names (`margin`, `color`, `nav`), and a lost `::before` line and a
   re-parented pairing (`divider`).
 - **Guard rails** — `blank`, `tall`, `animated`, `lazy`, `lazy-deep`, `lazy-js`, `lazy-grow`, `consent`,
-  `drifting`, `tracker`, `clock`, `ticker`, `unsettled`, `unsettled-once`, `sprite`, `short`, `tall-region`, `reorder`, `stripe`, `frameset`, `menu`, `scroll-spy`, `signed-in`, `fixed`, `inserted`, `labelled`, `broken-asset`, `media`, `long`, `video`, `video-seek`, `lottie`, `hang`, `origin`, `cooperate`, `mask-geometry`.
+  `drifting`, `tracker`, `clock`, `ticker`, `unsettled`, `unsettled-once`, `sprite`, `short`, `tall-region`, `reorder`, `stripe`, `frameset`, `menu`, `scroll-spy`, `signed-in`, `fixed`, `inserted`, `labelled`, `broken-asset`, `media`, `long`, `video`, `video-seek`, `lottie`, `hang`, `origin`, `cooperate`, `mask-geometry`, `bistable`.
   One per defence in `src/capture.ts` / `src/config.ts`.
 
 ## Authoring rules

tests/fixtures/pages/bistable/page.ts

@@ -0,0 +1,45 @@
+/**
+ * GUARD RAIL for `renderdiff probe`.
+ *
+ * A player box that is 366px tall on odd renders and 380px on even ones, the way an embed
+ * that sometimes gets its controls bar does. Every load is still, so a run only sees it as a
+ * base-vs-base difference; probe must name the box and its two heights. Generated because the
+ * counter has to live on the server.
+ */
+
+let renders = 0;
+
+const page = () => `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Layout parity</title>
+<link rel="stylesheet" href="/shared/base.css">
+<style>
+.player { height: ${++renders % 2 ? 366 : 380}px; background: var(--fill); border: 1px solid var(--line); }
+.band { height: 480px; padding: 16px; margin-top: var(--gap); background: var(--fill); border: 1px solid var(--line); overflow: hidden; }
+</style>
+</head>
+<body>
+<div class="page">
+<h1>Bistable guard rail</h1>
+<p class="lede">The box below is one of two heights, alternating per render. Each load is still, but two loads of the same page disagree.</p>
+
+<div class="player"></div>
+
+<section class="band">
+<h2>Static content</h2>
+<p>Everything else on this page is fixed, so the only thing that varies between loads is the player's height and what it pushes down.</p>
+</section>
+
+<section class="notes">
+<h2>Notes</h2>
+<p>A difference between two builds of the same base can come from the page itself. Loading one side several times shows which elements vary on their own.</p>
+</section>
+</div>
+</body>
+</html>
+`;
+
+export const left = page;
+export const right = page;

tests/helpers/corpus.ts

@@ -61,12 +61,13 @@ export const GUARDRAILS = [
   "origin",
   "cooperate",
   "mask-geometry",
+  "bistable",
 ] as const;
 
 export const ALL: string[] = [...EQUIVALENCE, ...CONTROLS, ...GUARDRAILS];
 
 /** Routes whose markup comes from a `page.ts` generator instead of static files. */
-export const GENERATED = ["tall", "drifting", "unsettled-once"];
+export const GENERATED = ["tall", "drifting", "unsettled-once", "bistable"];
 
 /** Routes that deliberately do not link `shared/base.css`. */
 export const SELF_CONTAINED = ["blank", "sprite", "tall", "frameset"];

tests/unit/cli-args.test.ts

@@ -396,6 +396,27 @@ describe("accept and rerun", () => {
   });
 });
 
+describe("probe", () => {
+  const U = `${L}/a`;
+  test("needs one URL", async () => {
+    await rejects(["probe"], "probe needs a URL");
+    await rejects(["probe", U, R], "probe takes one URL");
+    await rejects(["probe", U, "/b"], "probe takes one URL");
+  });
+
+  test("--loads is an integer of at least 2", async () => {
+    await rejects(["probe", U, "--loads", "1"], "--loads expects an integer of at least 2, got 1");
+    await rejects(["probe", U, "--loads=2.5"], "--loads expects an integer of at least 2");
+    await rejects(["probe", U, "--loads"], "--loads needs a value");
+    await rejects([L, R, "/a", "--loads", "3"], "unknown flag: --loads");
+  });
+
+  test("one width, no selector", async () => {
+    await rejects(["probe", U, "--width", "1440", "--width", "375"], "probe takes one --width");
+    await rejects(["probe", U, "--selector", "main"], "probe does not take --selector");
+  });
+});
+
 describe("preflight", () => {
   // Port 1 is reserved and never listening, so this is a closed port rather than a race
   // against something we just shut down.