From ecfbb38d13eed50a9b1a09649bf4b9dea867a59c Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:30:51 +0100 Subject: [PATCH 1/2] Add --aria: diff each route's ARIA snapshot per width Playwright's ARIA snapshot of both sides, diffed line by line after the shot, with asset hashes and the left origin normalized and masked elements left out. Pixels still decide the score and exit code. Fixes #45 Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + docs/output.md | 29 +++++ skills/renderdiff/SKILL.md | 2 +- skills/renderdiff/reference.md | 28 ++++- src/cli.ts | 3 + src/compare.ts | 2 +- src/computed.ts | 104 +++++++++++++++--- src/config.ts | 7 ++ src/index.ts | 2 +- src/report.ts | 13 +++ src/run.ts | 32 +++++- tests/fixtures/pages/README.md | 3 +- tests/fixtures/pages/button-vs-div/left.html | 27 +++++ tests/fixtures/pages/button-vs-div/right.html | 27 +++++ tests/helpers/corpus.ts | 1 + tests/unit/computed.test.ts | 56 ++++++++++ tests/visual/controls.test.ts | 28 +++++ 17 files changed, 342 insertions(+), 23 deletions(-) create mode 100644 tests/fixtures/pages/button-vs-div/left.html create mode 100644 tests/fixtures/pages/button-vs-div/right.html diff --git a/README.md b/README.md index 5d0bf80..a3439c4 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ renderdiff [options] | `--cache-right` | Replay the right side's network traffic from the last `--cache-right` run in `--out`, when it's under an hour old and covers every shot; otherwise load it live and record it. Keeps `/.cache/` when the out dir is wiped. See **Accepted differences and reruns**. | | `--no-recheck` | Don't reload a differing shot's right side to find content that varies per load. Off anyway under `--cache-right`. | | `--computed` | Also diff the computed style of every rendered element and its `::before`/`::after`, and say per route whether the markup is identical apart from asset hashes and origin. Pixels still decide the score and exit code. See [docs/output.md](docs/output.md#computed-styles---computed). | +| `--aria` | Also diff each route's ARIA snapshot (Playwright's accessibility tree) per width: left lines `+`, right `-`, asset hashes and origin normalized, masked elements left out. Pixels still decide. See [docs/output.md](docs/output.md#aria-snapshots---aria). | | `--sitemap[=]` | Add the paths in the right side's sitemap. Default `/sitemap.xml`; give another URL or path with `=`. | | `--dry-run` | Print the URL pairs and exit. No browser. | | `--open` | Open the report when the run finishes. | diff --git a/docs/output.md b/docs/output.md index 208b055..b72c7a2 100644 --- a/docs/output.md +++ b/docs/output.md @@ -60,6 +60,7 @@ the per-shot output when piped, `--no-progress` the summary on a terminal. | `1 accepted` | That many of the shot's changes are accepted; the rest are new. | | `findings: N repeated on 2+ routes` | After the routes: the same `dom` entry, text line or element-less band delta (`band -24px`) on two or more routes, once per line with `×N routes` and where (`/a@1440,375`, then `+N routes`), most widespread first, accepted ones aside. Top 20; `summary.findings` in results.json has them all. One cause behind many rows: fix it once. A selector names an element per page, so the same path may be a different element on another route: check one crop. | | `computed: N findings` | With `--computed`: each `key prop left→right` once, with the routes and widths it was seen at; `dom identical`: the markup matches apart from asset hashes and generated class names, so look at CSS or script. See [below](#computed-styles---computed). | +| `aria: identical on N of M route(s)` | With `--aria`: routes whose accessibility tree matches at every width, then each differing route's lines, `+` left only, `-` right only. See [below](#aria-snapshots---aria). | | `crops …` | Up to 3 PNGs per comparison, bands first, relative to `results.json`. Left is on top or on the left, and the change is boxed in red. | | `ERROR …` | Unmeasured, which is neither a pass nor a fail. Exits 3. | | `renderdiff: …` | The verdict line: routes and shots that differ (accepted ones aside), errors, the worst score, and with `--threshold` how many scored below it. | @@ -231,6 +232,33 @@ computed: 2 findings; dom identical on 2 of 3 route(s), differs /pricing (1), so `:hover`/`:focus` states aren't snapshotted. +## ARIA snapshots (`--aria`) + +With `--aria`, after the shot (and `--computed`) renderdiff takes Playwright's ARIA snapshot +of each side, the accessibility tree a screen reader gets, of `body` or the `--selector` +element, and diffs it line by line: + +``` +aria: identical on 2 of 3 route(s), differs /pricing + /pricing@1440,375 + +- text: Buy now + -- button "Buy now" +``` + +- `+` lines only the left (your change) has, `-` only the right. Above, a `div` styled like + the button it replaced: the same pixels, and the button role lost. +- The same diff at several widths is one group; a mobile-only nav change gets its own. + Ten lines per group, 20 groups; each result keeps up to 200 lines per side, and + `truncated` counts the rest. +- Masked elements are left out (marked `aria-hidden` once the shot is taken), and so are + bundler hashes in `/url:` values; the left origin reads as the right's, as with + `--computed`. +- Lines keep their indentation, so a landmark added or removed around content (`main` + swapped for a `div`) re-indents, and lists, every line under it. +- Iframes aren't descended into. A snapshot that fails or takes over 15s is reported as + failed, with why (`aria: null`, `ariaError`), and the pixel result stands. Pixels still + decide the score and the exit code. + ## Text A line diff of each side's visible text (`document.body.innerText`, or the element's with a @@ -347,6 +375,7 @@ pulls a route), scores rounded down to four decimals (so a shot that differs nev "failed": { "left": [], "right": [] }, // stylesheets and font faces that didn't load: "font Inter 700", "stylesheet https://…" "masked": ["rotating quote (data-renderdiff-mask)"], // absent unless a side marked elements data-renderdiff-mask "computed": { "dom": 1, "changes": [], "truncated": 0, "unmatched": { "left": [], "right": ["body > nav > a[href=\"/pricing\"]"] } }, // --computed only; null: snapshot failed, "computedError" says why + "aria": { "added": ["- text: Buy now"], "removed": ["- button \"Buy now\""], "truncated": 0 }, // --aria only; both empty: identical; null: snapshot failed, "ariaError" says why "images": { "left": "about.1440.left.png", "right": "about.1440.right.png", "diff": "about.1440.diff.png" }, "crops": [{ "path": "about.1440.c1.png", "kind": "band", // or "region" diff --git a/skills/renderdiff/SKILL.md b/skills/renderdiff/SKILL.md index 5e68763..275c7a0 100644 --- a/skills/renderdiff/SKILL.md +++ b/skills/renderdiff/SKILL.md @@ -20,7 +20,7 @@ the § named at each step. | Change | Run | | --- | --- | | CSS only, markup untouched (refactor, dead-CSS removal, token swap) | pixels + `--computed`. Expect `dom identical` on every route: then each `computed:` finding is a real style change, even on a clean route. `dom differs`: often a wrapper's class changed, re-keying its subtree (§ `--computed`). Hover, focus and open menus aren't shot without a `before` hook, and dead-CSS removal breaks exactly those. | -| Markup changes (framework, template or component migration) | pixels. `--computed` findings would only be hints. | +| Markup changes (framework, template or component migration) | pixels, plus `--aria`: `aria: identical` means a screen reader gets the same page. `--computed` findings would only be hints. | | Dependency or build upgrade | pixels, plus `--computed` if the markup should be identical. | | One component | pixels with `--selector ` or `'/route @sel'`. | diff --git a/skills/renderdiff/reference.md b/skills/renderdiff/reference.md index 96506a5..11f6eec 100644 --- a/skills/renderdiff/reference.md +++ b/skills/renderdiff/reference.md @@ -1,7 +1,7 @@ # renderdiff reference Left is your change, right is the reference: `+` is only/more on the left, `-` only/more on -the right. Sections: Reference and routes · Output · Errors · `--computed` · Masks, accepts +the right. Sections: Reference and routes · Output · Errors · `--computed` · `--aria` · Masks, accepts and thresholds · Rationalizations · Exit codes · Iterating · Scoping · Config file · results.json · Gotchas. @@ -49,6 +49,7 @@ accepted: /blog @375 ERROR HTTP 404 at http://localhost:3001/about findings: … (a change seen on 2+ routes) computed: … (with --computed, see below) +aria: … (with --aria, see below) renderdiff: 1 of 3 routes differ (1 of 15 shots), 1 error, worst 97.41% /about@1440 — …/results.json ``` @@ -170,6 +171,27 @@ computed: 2 findings; dom identical on 2 of 3 route(s), differs /pricing (1), so - Fails per shot (reported, pixels stand) past 15s or 50,000 elements. Not walked: SVG internals, shadow DOM, iframes, masked and `display: none` subtrees. +## `--aria` + +After each shot, diffs Playwright's ARIA snapshot (the accessibility tree) of `body` or the +`--selector` element, line by line: + +``` +aria: identical on 2 of 3 route(s), differs /pricing + /pricing@1440,375 + +- text: Buy now + -- button "Buy now" +``` + +- `+` only the left has, `-` only the right; the same diff at several widths is one group. + 10 lines per group shown, 200 per side kept in results.json. `--progress` rows say + `aria: N added, M removed`. +- Masked elements are left out, `/url:` asset hashes ignored, the left origin read as the + right's. Iframes aren't descended into. +- Indentation is kept, so a landmark added or removed around content re-lists every line + under it. +- Fails per shot past 15s (reported, pixels stand). + ## Masks, accepts and thresholds | Tool | Legitimate for | Never for | @@ -291,6 +313,7 @@ export default { freezeVideo: true, // --no-freeze-video sets false lottieFrame: 60, // hold lottie-player on frame 60; unset leaves it playing computed: false, // --computed + aria: false, // --aria }; ``` @@ -347,7 +370,8 @@ For scripts and drilling into one route; the loop doesn't need it. One result pe (`kind`, each side's box with `h` 0 at an insertion point, `layout`, `scale`, `clipped`); `accepted`; `rightReplayedFrom`; with `--computed`, `computed` (`dom`, 0 = identical; `changes` `{ key, prop, values }`, first 500; `truncated`; `unmatched`) or null with - `computedError`. + `computedError`; with `--aria`, `aria` (`added`, `removed`, first 200 each, both empty = + identical; `truncated`) or null with `ariaError`. - **`summary`:** `compared`, `errors`, `mean` and `worst` (accepted shots aside), `belowThreshold`, `differing` (unaccepted shots with any differing pixel; without `--threshold`, any exits 1), `accepted`, `stale`, `findings` (each `dom` entry, text line diff --git a/src/cli.ts b/src/cli.ts index d56191a..bdd9283 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -98,6 +98,7 @@ function parseArgs(argv: string[]): Args { else if (v === "--cache-right") a.cacheRight = true; else if (v === "--no-recheck") a.recheck = false; else if (v === "--computed") a.computed = true; + else if (v === "--aria") a.aria = true; else if (v === "--mask-media") a.maskMedia = true; else if (v === "--no-freeze-video") a.freezeVideo = false; else if (v === "--sitemap") { @@ -250,6 +251,8 @@ const HELP = ` varies per load. Off anyway under --cache-right. --computed Also diff every element's computed style and ::before/::after, and say whether the markup is identical. Pixels still decide. + --aria Also diff each page's accessibility tree (Playwright's ARIA + snapshot) line by line. Pixels still decide. --sitemap[=] Add the paths in the right side's sitemap (one index level, files dropped). Default /sitemap.xml; give a URL or path with =. --dry-run Print the URL pairs and exit. No browser. diff --git a/src/compare.ts b/src/compare.ts index 7d8030e..8745120 100644 --- a/src/compare.ts +++ b/src/compare.ts @@ -252,7 +252,7 @@ function backtrack(trace: Int32Array[], n: number, m: number, D: number) { return pairs.reverse(); } -function textDiff(a: string[], b: string[]): TextDiff { +export function textDiff(a: string[], b: string[]): TextDiff { const pairs = myers(a, b); if (!pairs) { // Too different to align: fall back to plain membership. diff --git a/src/computed.ts b/src/computed.ts index b9c5f2b..a15584f 100644 --- a/src/computed.ts +++ b/src/computed.ts @@ -1,3 +1,4 @@ +import { textDiff } from "./compare.ts"; import { name } from "./config.ts"; import { PROPS } from "./locate.ts"; import type { ComparedResult, RouteResult } from "./report.ts"; @@ -34,7 +35,17 @@ export interface Finding extends StyleChange { at: { url: string; widths: number[] }[]; } +export interface AriaDiff { + /** Lines of Playwright's ARIA snapshot only the left (your change) has, at most `MAX_ARIA_LINES`; both empty: identical. */ + added: string[]; + removed: string[]; + /** Lines past `MAX_ARIA_LINES` on either side, left out. */ + truncated: number; +} + const MAX_CHANGES = 500; +const MAX_ARIA_LINES = 200; +const ARIA_LINES_SHOWN = 10; const MAX_UNMATCHED = 20; const FIELDS = [...PROPS, "content"]; @@ -66,16 +77,19 @@ const hashy = (t: string) => (t.match(/\d+|[A-Za-z]+/g)?.length ?? 0) >= 4; /** In URL attributes, bundler hashes in a file name (`main.3f9a8c1d.png`, `?v=…`) read the same on both sides. */ export const normalizeAssets = (html: string) => + html.replace( + / (src|href|srcset|poster)="([^"]*)"/g, + (_, n: string, v: string) => ` ${n}="${normalizeUrl(v)}"`, + ); + +const normalizeUrl = (v: string) => // ponytail: hex/base36 hashes between separators; add a framework's build-id path when one shows up - html.replace(/ (src|href|srcset|poster)="([^"]*)"/g, (_, n: string, v: string) => { - const u = v - .replace( - /([.\-_])([A-Za-z0-9]{8,})(?=(?:[.\-_][A-Za-z0-9]+)*\.[A-Za-z0-9]{2,5}(?:$|[?#\s,]))/g, - (m, sep: string, t: string) => (hashy(t) ? `${sep}#` : m), - ) - .replace(/([?&](?:v|ver|version|hash)=)[\w.-]+/g, "$1#"); - return ` ${n}="${u}"`; - }); + v + .replace( + /([.\-_])([A-Za-z0-9]{8,})(?=(?:[.\-_][A-Za-z0-9]+)*\.[A-Za-z0-9]{2,5}(?:$|[?#\s,]))/g, + (m, sep: string, t: string) => (hashy(t) ? `${sep}#` : m), + ) + .replace(/([?&](?:v|ver|version|hash)=)[\w.-]+/g, "$1#"); /** A page's throw is its text after the first `Error: `, without the stack page.evaluate appends. */ export const pageError = (e: unknown) => { @@ -115,19 +129,20 @@ export function collapseInherited x.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // Not `:46110` for `:4611`, nor `localhost.example.com` for `localhost`. const re = new RegExp(`(${esc(f.protocol)})?//${esc(f.host)}(?![\\w.:-])`, "g"); - // Origins hold no `"` or `\`, so the JSON text rewrites like the strings it encodes. - return JSON.parse( - JSON.stringify(s).replace(re, (_, p?: string) => `${p ? t.protocol : ""}//${t.host}`), - ) as Snapshot; + return s.replace(re, (_, p?: string) => `${p ? t.protocol : ""}//${t.host}`); } +/** In keys, markup and styles. Origins hold no `"` or `\`, so the JSON text rewrites like the strings it encodes. */ +const rebase = (s: Snapshot, from: string, to: string): Snapshot => + JSON.parse(rebaseText(JSON.stringify(s), from, to)) as Snapshot; + export function diffSnapshots( left: Snapshot, right: Snapshot, @@ -279,3 +294,62 @@ export function fmtComputed(results: RouteResult[], routes: string[]): string[] out.push(` (+${unmatched.length - UNMATCHED_SHOWN} more routes with unmatched elements)`); return out; } + +// ponytail: raw indented lines, so a wrapper role added or removed re-indents everything below it; key lines by role path if that gets noisy +export function diffAria(left: string, right: string, origins?: [string, string]): AriaDiff { + const lines = (t: string) => + t.split("\n").map((l) => l.replace(/^(\s*- \/url: )(.*)$/, (_, p, u) => p + normalizeUrl(u))); + const { added, removed } = textDiff( + lines(origins ? rebaseText(left, ...origins) : left), + lines(right), + ); + return { + added: added.slice(0, MAX_ARIA_LINES), + removed: removed.slice(0, MAX_ARIA_LINES), + truncated: + Math.max(0, added.length - MAX_ARIA_LINES) + Math.max(0, removed.length - MAX_ARIA_LINES), + }; +} + +/** The console block: identical routes counted, then each distinct diff once with the widths it was seen at. */ +export function fmtAria(results: RouteResult[], routes: string[]): string[] { + const rows = results.filter((r): r is ComparedResult => r.status === "compared" && "aria" in r); + if (!rows.length) return []; + const groups = new Map(); + const failed: string[] = []; + const differs: string[] = []; + let identical = 0; + for (const route of routes) { + const rs = rows.filter((r) => name(r) === route); + if (!rs.length) continue; + const bad = rs.find((r) => !r.aria); + if (bad) failed.push(`${route}@${bad.width} (${bad.ariaError ?? "unknown"})`); + const diffs = rs.filter((r) => r.aria && (r.aria.added.length || r.aria.removed.length)); + if (diffs.length) differs.push(route); + else if (rs.some((r) => r.aria)) identical++; + for (const r of diffs) { + const { added, removed, truncated } = r.aria!; + const id = JSON.stringify([route, added, removed]); + const g = groups.get(id); + if (g) g.widths.push(r.width); + else { + const lines = [...added.map((l) => `+${l}`), ...removed.map((l) => `-${l}`)]; + groups.set(id, { at: route, widths: [r.width], lines, more: truncated }); + } + } + } + const out = [ + `aria: identical on ${identical} of ${identical + differs.length} route(s)` + + (differs.length ? `, differs ${differs.join(" ")}` : "") + + (failed.length ? `; snapshot failed on ${failed.join(" ")}` : ""), + ]; + const all = [...groups.values()]; + for (const g of all.slice(0, SHOWN)) { + out.push(` ${g.at}@${g.widths.join(",")}`); + out.push(...g.lines.slice(0, ARIA_LINES_SHOWN).map((l) => ` ${l}`)); + const more = Math.max(0, g.lines.length - ARIA_LINES_SHOWN) + g.more; + if (more) out.push(` (+${more} more in results.json)`); + } + if (all.length > SHOWN) out.push(` (+${all.length - SHOWN} more in results.json)`); + return out; +} diff --git a/src/config.ts b/src/config.ts index 0127b74..6a6adf6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -292,6 +292,8 @@ export interface RenderdiffConfig { masks?: Mask[]; /** Also diff every element's computed style, and say whether the markup is identical. */ computed?: boolean; + /** Also diff each page's ARIA snapshot. */ + aria?: boolean; /** Mask every `iframe` on every route. */ maskMedia?: boolean; /** Keep every video paused on frame 0 (its poster if it never loads) and hide video.js overlays. Default true. */ @@ -320,6 +322,8 @@ export interface ResolvedConfig { freezeVideo: boolean; /** Optional, not null, so an unset value leaves the config hash unchanged. */ lottieFrame?: number; + /** Optional like `lottieFrame`. */ + aria?: true; } /** package.json's version; the same relative path resolves from `src/` and `dist/`. */ @@ -492,6 +496,7 @@ function validate(c: Record, file: string) { } } if (c.computed !== undefined && typeof c.computed !== "boolean") bad("computed", "a boolean"); + if (c.aria !== undefined && typeof c.aria !== "boolean") bad("aria", "a boolean"); if (c.threshold !== undefined) { const t = c.threshold; if (typeof t !== "number" || !Number.isFinite(t) || t < 0 || t > 1) { @@ -548,6 +553,7 @@ export interface CliOverrides { /** Per side, from --storage-state / --left-storage-state / --right-storage-state. */ storageState?: { left?: string; right?: string }; computed?: boolean; + aria?: boolean; /** From --mask, appended to the config's masks. */ masks?: string[]; maskMedia?: boolean; @@ -609,6 +615,7 @@ export function resolveConfig(file: RenderdiffConfig, cli: CliOverrides): Resolv computed: cli.computed ?? file.computed ?? false, freezeVideo, lottieFrame: file.lottieFrame, + aria: (cli.aria ?? file.aria) || undefined, }; } diff --git a/src/index.ts b/src/index.ts index 1934dee..c8299e5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -38,5 +38,5 @@ export { export { type Band, type Box, type Region, type TextDiff } from "./compare.ts"; export { type ElementDiff } from "./locate.ts"; -export { type ComputedDiff, type Finding, type StyleChange } from "./computed.ts"; +export { type AriaDiff, type ComputedDiff, type Finding, type StyleChange } from "./computed.ts"; export { type Accepted, type Fingerprint } from "./accept.ts"; diff --git a/src/report.ts b/src/report.ts index 5f7d868..cd0e0bf 100644 --- a/src/report.ts +++ b/src/report.ts @@ -5,9 +5,11 @@ import type { Accepted } from "./accept.ts"; import { addAt, fmtAt, + fmtAria, fmtComputed, groupFindings, SHOWN, + type AriaDiff, type ComputedDiff, type Finding, } from "./computed.ts"; @@ -77,6 +79,9 @@ export interface ComparedResult extends BaseResult { /** With `--computed`: the computed-style diff; null when the snapshot failed, with `computedError` saying why. */ computed?: ComputedDiff | null; computedError?: string; + /** With `--aria`: the ARIA snapshot diff; null when the snapshot failed, with `ariaError` saying why. */ + aria?: AriaDiff | null; + ariaError?: string; } /** A crop of one change: left and right panels, a 1px red box round it, or a 2px red seam. */ @@ -326,6 +331,7 @@ function summary(results: RouteResult[], meta: RunMeta, stale: Accepted[]): stri const byRoute = Map.groupBy(stats(differing).sorted, name); const routes = [...new Set(results.map(name))]; const computed = fmtComputed(results, routes); + const aria = fmtAria(results, routes); const repeated = fmtPixelFindings(results); return `

@@ -367,6 +373,12 @@ ${

${esc(computed.join("\n"))}
` : "" } +${ + aria.length + ? `
ARIA: ${esc(aria[0]!.slice("aria: ".length))} +
${esc(aria.slice(1).join("\n"))}
` + : "" +}
`; } @@ -435,6 +447,7 @@ const head = (r: ComparedResult) => ` ${r.variesPixels ? `${r.variesPixels.toLocaleString("en-US")} px vary per load` : ""} ${r.alignedScore !== null ? `aligned ${pct(r.alignedScore)}` : ""} ${r.computed ? `${r.computed.dom ? `dom differs (${r.computed.dom})` : "dom identical"}` : ""} + ${r.aria ? `${r.aria.added.length || r.aria.removed.length ? `aria differs (+${r.aria.added.length} −${r.aria.removed.length})` : "aria identical"}` : ""} ${r.diffPixels.toLocaleString()} / ${r.totalPixels.toLocaleString()} px differ `; diff --git a/src/run.ts b/src/run.ts index a3f3646..a4d24e2 100644 --- a/src/run.ts +++ b/src/run.ts @@ -9,7 +9,7 @@ import { applyAccepted, readAccepted, unacceptable, updateAccepted } from "./acc import { CACHE_DIR, cacheFile, replayable, REPLAY_TTL, saveRecording, shotId } from "./cache.ts"; import { cancelUnstable, capture, HookError, prepare, step, type Shot } from "./capture.ts"; import { compare, crops, type Band, type Box, type Region } from "./compare.ts"; -import { diffSnapshots, fmtComputed, pageError } from "./computed.ts"; +import { diffAria, diffSnapshots, fmtAria, fmtComputed, pageError } from "./computed.ts"; import { locate, snapshot, type ElementDiff } from "./locate.ts"; import { CONTEXT_OPTIONS, @@ -170,7 +170,7 @@ function previous(file: string, now: Record): Results | string for (const [key, value] of Object.entries(now)) { if (key === "config" && value !== was[key]) { return ( - `the config (masks, hooks, hidden selectors, blocked URLs, CSS, storage state, --computed) ` + + `the config (masks, hooks, hidden selectors, blocked URLs, CSS, storage state, --computed, --aria) ` + `differs from the one that wrote ${file}; run without --rerun-failed to start over` ); } @@ -478,6 +478,26 @@ export async function run({ })), ]) : null; + // Masked content is noise here too; every next job reloads the page, so no restore. + const tree = async (page: Page, side: "left" | "right") => { + for (const m of masksFor(cfg, page, route)) + await m.evaluateAll((es) => es.forEach((e) => e.setAttribute("aria-hidden", "true"))); + return page + .locator(route.selector?.[side] ?? "body") + .ariaSnapshot({ timeout: COMPUTED_TIMEOUT }); + }; + const aria = cfg.aria + ? await Promise.race([ + Promise.all([tree(pages.left, "left"), tree(pages.right, "right")]).then( + ([l, r]) => ({ aria: diffAria(l, r, [pages.left.url(), pages.right.url()]) }), + (e: unknown) => ({ aria: null, ariaError: pageError(e) }), + ), + sleep(COMPUTED_TIMEOUT + 5_000, null, { ref: false }).then(() => ({ + aria: null, + ariaError: `took over ${(COMPUTED_TIMEOUT + 5_000) / 1000}s`, + })), + ]) + : null; const cropped = made.map(({ png, ...c }, i) => ({ path: `${base}.c${i + 1}.png`, ...c })); await Promise.all([ write(images.left, leftShot.png), @@ -500,6 +520,7 @@ export async function run({ failed: { left: leftShot.failed, right: rightShot.failed }, ...(marked.length ? { masked: marked } : {}), ...computed, + ...aria, ...(replayedAt ? { rightReplayedFrom: new Date(replayedAt).toISOString() } : {}), }); const { regions, bands, alignedScore, text } = metrics; @@ -525,6 +546,10 @@ export async function run({ computed?.computed ? `computed ${computed.computed.dom ? `dom differs (${computed.computed.dom})` : "dom identical"}, ${computed.computed.changes.length + computed.computed.truncated} change(s)` : "", + aria && !aria.aria ? `aria: snapshot failed, ${aria.ariaError}` : "", + aria?.aria && (aria.aria.added.length || aria.aria.removed.length) + ? `aria: ${aria.aria.added.length} added, ${aria.aria.removed.length} removed` + : "", ].filter(Boolean); ++done; if (!progress) continue; @@ -935,6 +960,8 @@ function summarize( if (repeated.length) console.log(`\n${repeated.join("\n")}`); const computed = fmtComputed(results, routes); if (computed.length) console.log(`\n${computed.join("\n")}`); + const aria = fmtAria(results, routes); + if (aria.length) console.log(`\n${aria.join("\n")}`); const varies = fmtVaries(results, routes); if (varies.length) console.log(`\n${varies.join("\n")}`); @@ -1032,6 +1059,7 @@ export function compact( out.push( ...fmtPixelFindings(results), ...fmtComputed(results, routes), + ...fmtAria(results, routes), ...fmtVaries(results, routes), ); const all = results.filter((r): r is ComparedResult => r.status === "compared"); diff --git a/tests/fixtures/pages/README.md b/tests/fixtures/pages/README.md index b3bd65e..624d6b4 100644 --- a/tests/fixtures/pages/README.md +++ b/tests/fixtures/pages/README.md @@ -9,7 +9,8 @@ hand. Three groups: - **Equivalence** — `home`, `flex-vs-grid`, `float-vs-flex`, `table-vs-grid`, `abs-vs-flex`, - `inline-block-vs-flex`. The two sides are visually identical and built with different CSS. + `inline-block-vs-flex`, `button-vs-div`. The two sides are visually identical and built with + different CSS (`button-vs-div`: different markup, for `--aria`). These must score ≈100%. They are the corpus's central claim: the score tracks pixels, not markup, so a CSS refactor that changed nothing visible reads as "nothing changed". - **Controls** — `shifted`, `taller`, `color`, `tint`, `nav`, `margin`, `divider`. Deliberately different. These must score low. diff --git a/tests/fixtures/pages/button-vs-div/left.html b/tests/fixtures/pages/button-vs-div/left.html new file mode 100644 index 0000000..6683aad --- /dev/null +++ b/tests/fixtures/pages/button-vs-div/left.html @@ -0,0 +1,27 @@ + + + + +Layout parity + + + + +
+

Layout parity check

+

Both sides of this pair render the same pixels. One side's call to action is a button, the other's a plain block, so only the accessibility tree tells them apart.

+ +

Ready to order?

+
Buy now
+ +
+

Notes

+

The geometry here is deliberately integral: fixed widths, fixed heights, and no fractional boxes anywhere. Every inherited style is reset on both elements, so the button and the block draw the same box.

+

Text is identical on both sides. Nothing loads from the network except assets served by the fixture origin itself, so this page renders the same offline as it does in continuous integration.

+
+
+ + diff --git a/tests/fixtures/pages/button-vs-div/right.html b/tests/fixtures/pages/button-vs-div/right.html new file mode 100644 index 0000000..132e5f3 --- /dev/null +++ b/tests/fixtures/pages/button-vs-div/right.html @@ -0,0 +1,27 @@ + + + + +Layout parity + + + + +
+

Layout parity check

+

Both sides of this pair render the same pixels. One side's call to action is a button, the other's a plain block, so only the accessibility tree tells them apart.

+ +

Ready to order?

+ + +
+

Notes

+

The geometry here is deliberately integral: fixed widths, fixed heights, and no fractional boxes anywhere. Every inherited style is reset on both elements, so the button and the block draw the same box.

+

Text is identical on both sides. Nothing loads from the network except assets served by the fixture origin itself, so this page renders the same offline as it does in continuous integration.

+
+
+ + diff --git a/tests/helpers/corpus.ts b/tests/helpers/corpus.ts index 76a699b..ba5e15b 100644 --- a/tests/helpers/corpus.ts +++ b/tests/helpers/corpus.ts @@ -18,6 +18,7 @@ export const EQUIVALENCE = [ "table-vs-grid", "abs-vs-flex", "inline-block-vs-flex", + "button-vs-div", ] as const; /** Deliberately different. Must score low — otherwise the suite proves nothing. */ diff --git a/tests/unit/computed.test.ts b/tests/unit/computed.test.ts index 4b0ac2c..728ebf6 100644 --- a/tests/unit/computed.test.ts +++ b/tests/unit/computed.test.ts @@ -6,7 +6,9 @@ import assert from "node:assert/strict"; import { describe, test } from "node:test"; import { collapseInherited, + diffAria, diffSnapshots, + fmtAria, fmtComputed, groupFindings, normalizeAssets, @@ -315,3 +317,57 @@ test("a snapshot error keeps the page's message, not page.evaluate's prefix or s "locator.evaluate: Timeout 30000ms exceeded.\nCall log:\n - waiting for locator('.x')"; assert.equal(pageError(new Error(timeout)), timeout); }); + +describe("diffAria", () => { + const same = { added: [], removed: [], truncated: 0 }; + + test("asset hashes and the other side's origin read the same", () => { + const tree = (css: string, abs: string) => + `- link "Styles":\n - /url: ${css}\n- link "Home":\n - /url: ${abs}/x`; + assert.deepEqual( + diffAria( + tree("/main.3f9a8c1d.css?v=abc", "http://localhost:4611"), + tree("/main.9b2e7f04.css?v=def", "http://localhost:4612"), + ["http://localhost:4611/a", "http://localhost:4612/a"], + ), + same, + ); + }); + + test("a lost role is one line each way; past the cap only a count is kept", () => { + assert.deepEqual(diffAria("- p: Hi\n- text: Buy", '- p: Hi\n- button "Buy"'), { + added: ["- text: Buy"], + removed: ['- button "Buy"'], + truncated: 0, + }); + const many = (s: string) => Array.from({ length: 205 }, (_, i) => `- ${s} ${i}`).join("\n"); + const d = diffAria(many("a"), many("b")); + assert.deepEqual([d.added.length, d.removed.length, d.truncated], [200, 200, 10]); + }); + + test("fmtAria shows a diff once across widths and splits one that differs", () => { + const lost = { added: ["- text: Buy"], removed: ['- button "Buy"'], truncated: 0 }; + assert.deepEqual( + fmtAria( + [ + shot("/a", 1440, { aria: lost }), + shot("/a", 375, { aria: lost }), + shot("/b", 1440, { aria: same }), + shot("/b", 375, { aria: { ...same, added: ["- navigation"] } }), + shot("/c", 1440, { aria: same }), + shot("/d", 1440, { aria: null, ariaError: "took over 20s" }), + ], + ["/a", "/b", "/c", "/d"], + ), + [ + "aria: identical on 1 of 3 route(s), differs /a /b; snapshot failed on /d@1440 (took over 20s)", + " /a@1440,375", + " +- text: Buy", + ' -- button "Buy"', + " /b@375", + " +- navigation", + ], + ); + assert.deepEqual(fmtAria([shot("/a", 1440, {})], ["/a"]), []); + }); +}); diff --git a/tests/visual/controls.test.ts b/tests/visual/controls.test.ts index 690d20d..711bfd6 100644 --- a/tests/visual/controls.test.ts +++ b/tests/visual/controls.test.ts @@ -603,3 +603,31 @@ test("--computed names the changed rule's property, once across widths, and the ); assert.match(r.stdout, /a\[href="#blog"\] margin-left 24px→16px {2}\/margin@1440,375/); }); + +test("--aria names a div that lost its button role, where pixels see nothing", async () => { + const r = await runCli([ + ...renderdiffArgs({ + left: site.left, + right: site.right, + routes: ["/button-vs-div", "/origin", "/margin"], + widths: WIDTHS, + }), + "--aria", + ]); + assert.equal(r.code, 1, context(r)); + const { results: rows } = results(r.outDir); + const aria = (route: string) => + rows.filter((x) => x.url === route).map((x) => x.status === "compared" && x.aria); + const at = (route: string) => rows.filter((x) => x.url === route && x.status === "compared"); + assert.deepEqual( + at("/button-vs-div").map((x) => x.status === "compared" && x.score), + [1, 1], + ); + const lost = { added: ["- text: Buy now"], removed: ['- button "Buy now"'], truncated: 0 }; + assert.deepEqual(aria("/button-vs-div"), [lost, lost], context(r)); + const same = { added: [], removed: [], truncated: 0 }; + assert.deepEqual(aria("/origin"), [same, same], context(r)); + assert.deepEqual(aria("/margin"), [same, same], context(r)); + assert.match(r.stdout, /^aria: identical on 2 of 3 route\(s\), differs \/button-vs-div/m); + assert.equal(r.stdout.match(/\/button-vs-div@1440,375/g)?.length, 1, context(r)); +}); From 5433ffb5b8a4dfb3870f7088d97a7d01e1b18901 Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:39:34 +0100 Subject: [PATCH 2/2] Test --aria's mask exclusion and @selector scope; add aria to the example config Co-Authored-By: Claude Opus 5.5 --- renderdiff.config.example.ts | 4 ++++ tests/visual/controls.test.ts | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/renderdiff.config.example.ts b/renderdiff.config.example.ts index f383852..31df8a0 100644 --- a/renderdiff.config.example.ts +++ b/renderdiff.config.example.ts @@ -95,6 +95,10 @@ export default { // apart from asset hashes. Same as --computed. Pixels still decide the score. computed: false, + // Also diff each page's ARIA snapshot (the accessibility tree). Same as --aria. + // Pixels still decide the score. + aria: false, + // Content that renders on both sides but whose value drifts. Painted over on both, // so the noise cancels out instead of scoring against you. // diff --git a/tests/visual/controls.test.ts b/tests/visual/controls.test.ts index 711bfd6..55d4ab4 100644 --- a/tests/visual/controls.test.ts +++ b/tests/visual/controls.test.ts @@ -631,3 +631,23 @@ test("--aria names a div that lost its button role, where pixels see nothing", a assert.match(r.stdout, /^aria: identical on 2 of 3 route\(s\), differs \/button-vs-div/m); assert.equal(r.stdout.match(/\/button-vs-div@1440,375/g)?.length, 1, context(r)); }); + +test("--aria leaves out masked elements and reads only a route's @selector element", async () => { + const runs = await Promise.all( + [["/button-vs-div", "--mask", ".buy"], ["/button-vs-div @.notes"]].map(([route, ...flags]) => + runCli([ + ...renderdiffArgs({ left: site.left, right: site.right, routes: [route!], widths: [1440] }), + "--aria", + ...flags, + ]), + ), + ); + for (const r of runs) { + const [row] = results(r.outDir).results; + assert.deepEqual( + row?.status === "compared" && row.aria, + { added: [], removed: [], truncated: 0 }, + context(r), + ); + } +});