Add --aria: diff ARIA snapshots per route

README.md

@@ -115,6 +115,7 @@ renderdiff <left-origin> <right-origin> <route…> [options]
 | `--cache-right` | Replay the right side's network traffic from the last `--cache-right` run in `--out`, when it's under an hour old and covers every shot; otherwise load it live and record it. Keeps `<out>/.cache/` when the out dir is wiped. See **Accepted differences and reruns**. |
 | `--no-recheck` | Don't reload a differing shot's right side to find content that varies per load. Off anyway under `--cache-right`. |
 | `--computed` | Also diff the computed style of every rendered element and its `::before`/`::after`, and say per route whether the markup is identical apart from asset hashes and origin. Pixels still decide the score and exit code. See [docs/output.md](docs/output.md#computed-styles---computed). |
+| `--aria` | Also diff each route's ARIA snapshot (Playwright's accessibility tree) per width: left lines `+`, right `-`, asset hashes and origin normalized, masked elements left out. Pixels still decide. See [docs/output.md](docs/output.md#aria-snapshots---aria). |
 | `--sitemap[=<url>]` | Add the paths in the right side's sitemap. Default `<right>/sitemap.xml`; give another URL or path with `=`. |
 | `--dry-run` | Print the URL pairs and exit. No browser. |
 | `--open` | Open the report when the run finishes. |

docs/output.md

@@ -60,6 +60,7 @@ the per-shot output when piped, `--no-progress` the summary on a terminal.
 | `1 accepted` | That many of the shot's changes are accepted; the rest are new. |
 | `findings: N repeated on 2+ routes` | After the routes: the same `dom` entry, text line or element-less band delta (`band -24px`) on two or more routes, once per line with `×N routes` and where (`/a@1440,375`, then `+N routes`), most widespread first, accepted ones aside. Top 20; `summary.findings` in results.json has them all. One cause behind many rows: fix it once. A selector names an element per page, so the same path may be a different element on another route: check one crop. |
 | `computed: N findings` | With `--computed`: each `key prop left→right` once, with the routes and widths it was seen at; `dom identical`: the markup matches apart from asset hashes and generated class names, so look at CSS or script. See [below](#computed-styles---computed). |
+| `aria: identical on N of M route(s)` | With `--aria`: routes whose accessibility tree matches at every width, then each differing route's lines, `+` left only, `-` right only. See [below](#aria-snapshots---aria). |
 | `crops …` | Up to 3 PNGs per comparison, bands first, relative to `results.json`. Left is on top or on the left, and the change is boxed in red. |
 | `ERROR …` | Unmeasured, which is neither a pass nor a fail. Exits 3. |
 | `renderdiff: …` | The verdict line: routes and shots that differ (accepted ones aside), errors, the worst score, and with `--threshold` how many scored below it. |
@@ -231,6 +232,33 @@ computed: 2 findings; dom identical on 2 of 3 route(s), differs /pricing (1), so
 
 `:hover`/`:focus` states aren't snapshotted.
 
+## ARIA snapshots (`--aria`)
+
+With `--aria`, after the shot (and `--computed`) renderdiff takes Playwright's ARIA snapshot
+of each side, the accessibility tree a screen reader gets, of `body` or the `--selector`
+element, and diffs it line by line:
+
+```
+aria: identical on 2 of 3 route(s), differs /pricing
+  /pricing@1440,375
+    +- text: Buy now
+    -- button "Buy now"
+```
+
+- `+` lines only the left (your change) has, `-` only the right. Above, a `div` styled like
+  the button it replaced: the same pixels, and the button role lost.
+- The same diff at several widths is one group; a mobile-only nav change gets its own.
+  Ten lines per group, 20 groups; each result keeps up to 200 lines per side, and
+  `truncated` counts the rest.
+- Masked elements are left out (marked `aria-hidden` once the shot is taken), and so are
+  bundler hashes in `/url:` values; the left origin reads as the right's, as with
+  `--computed`.
+- Lines keep their indentation, so a landmark added or removed around content (`main`
+  swapped for a `div`) re-indents, and lists, every line under it.
+- Iframes aren't descended into. A snapshot that fails or takes over 15s is reported as
+  failed, with why (`aria: null`, `ariaError`), and the pixel result stands. Pixels still
+  decide the score and the exit code.
+
 ## Text
 
 A line diff of each side's visible text (`document.body.innerText`, or the element's with a
@@ -347,6 +375,7 @@ pulls a route), scores rounded down to four decimals (so a shot that differs nev
       "failed": { "left": [], "right": [] },   // stylesheets and font faces that didn't load: "font Inter 700", "stylesheet https://…"
       "masked": ["rotating quote (data-renderdiff-mask)"],   // absent unless a side marked elements data-renderdiff-mask
       "computed": { "dom": 1, "changes": [], "truncated": 0, "unmatched": { "left": [], "right": ["body > nav > a[href=\"/pricing\"]"] } },   // --computed only; null: snapshot failed, "computedError" says why
+      "aria": { "added": ["- text: Buy now"], "removed": ["- button \"Buy now\""], "truncated": 0 },   // --aria only; both empty: identical; null: snapshot failed, "ariaError" says why
       "images": { "left": "about.1440.left.png", "right": "about.1440.right.png", "diff": "about.1440.diff.png" },
       "crops": [{
         "path": "about.1440.c1.png", "kind": "band",          // or "region"

renderdiff.config.example.ts

@@ -95,6 +95,10 @@ export default {
   // apart from asset hashes. Same as --computed. Pixels still decide the score.
   computed: false,
 
+  // Also diff each page's ARIA snapshot (the accessibility tree). Same as --aria.
+  // Pixels still decide the score.
+  aria: false,
+
   // Content that renders on both sides but whose value drifts. Painted over on both,
   // so the noise cancels out instead of scoring against you.
   //

skills/renderdiff/SKILL.md

@@ -20,7 +20,7 @@ the § named at each step.
 | Change | Run |
 | --- | --- |
 | CSS only, markup untouched (refactor, dead-CSS removal, token swap) | pixels + `--computed`. Expect `dom identical` on every route: then each `computed:` finding is a real style change, even on a clean route. `dom differs`: often a wrapper's class changed, re-keying its subtree (§ `--computed`). Hover, focus and open menus aren't shot without a `before` hook, and dead-CSS removal breaks exactly those. |
-| Markup changes (framework, template or component migration) | pixels. `--computed` findings would only be hints. |
+| Markup changes (framework, template or component migration) | pixels, plus `--aria`: `aria: identical` means a screen reader gets the same page. `--computed` findings would only be hints. |
 | Dependency or build upgrade | pixels, plus `--computed` if the markup should be identical. |
 | One component | pixels with `--selector <sel>` or `'/route @sel'`. |
 

skills/renderdiff/reference.md

@@ -1,7 +1,7 @@
 # renderdiff reference
 
 Left is your change, right is the reference: `+` is only/more on the left, `-` only/more on
-the right. Sections: Reference and routes · Output · Errors · `--computed` · Masks, accepts
+the right. Sections: Reference and routes · Output · Errors · `--computed` · `--aria` · Masks, accepts
 and thresholds · Rationalizations · Exit codes · Iterating · Scoping · Config file ·
 results.json · Gotchas.
 
@@ -49,6 +49,7 @@ accepted: /blog
   @375  ERROR HTTP 404 at http://localhost:3001/about
 findings: …                       (a change seen on 2+ routes)
 computed: …                       (with --computed, see below)
+aria: …                           (with --aria, see below)
 renderdiff: 1 of 3 routes differ (1 of 15 shots), 1 error, worst 97.41% /about@1440 — …/results.json
 ```
 
@@ -170,6 +171,27 @@ computed: 2 findings; dom identical on 2 of 3 route(s), differs /pricing (1), so
 - Fails per shot (reported, pixels stand) past 15s or 50,000 elements. Not walked: SVG
   internals, shadow DOM, iframes, masked and `display: none` subtrees.
 
+## `--aria`
+
+After each shot, diffs Playwright's ARIA snapshot (the accessibility tree) of `body` or the
+`--selector` element, line by line:
+
+```
+aria: identical on 2 of 3 route(s), differs /pricing
+  /pricing@1440,375
+    +- text: Buy now
+    -- button "Buy now"
+```
+
+- `+` only the left has, `-` only the right; the same diff at several widths is one group.
+  10 lines per group shown, 200 per side kept in results.json. `--progress` rows say
+  `aria: N added, M removed`.
+- Masked elements are left out, `/url:` asset hashes ignored, the left origin read as the
+  right's. Iframes aren't descended into.
+- Indentation is kept, so a landmark added or removed around content re-lists every line
+  under it.
+- Fails per shot past 15s (reported, pixels stand).
+
 ## Masks, accepts and thresholds
 
 | Tool | Legitimate for | Never for |
@@ -291,6 +313,7 @@ export default {
   freezeVideo: true,    // --no-freeze-video sets false
   lottieFrame: 60,      // hold lottie-player on frame 60; unset leaves it playing
   computed: false,      // --computed
+  aria: false,          // --aria
 };
 ```
 
@@ -347,7 +370,8 @@ For scripts and drilling into one route; the loop doesn't need it. One result pe
   (`kind`, each side's box with `h` 0 at an insertion point, `layout`, `scale`, `clipped`);
   `accepted`; `rightReplayedFrom`; with `--computed`, `computed` (`dom`, 0 = identical;
   `changes` `{ key, prop, values }`, first 500; `truncated`; `unmatched`) or null with
-  `computedError`.
+  `computedError`; with `--aria`, `aria` (`added`, `removed`, first 200 each, both empty =
+  identical; `truncated`) or null with `ariaError`.
 - **`summary`:** `compared`, `errors`, `mean` and `worst` (accepted shots aside),
   `belowThreshold`, `differing` (unaccepted shots with any differing pixel; without
   `--threshold`, any exits 1), `accepted`, `stale`, `findings` (each `dom` entry, text line

src/cli.ts

@@ -98,6 +98,7 @@ function parseArgs(argv: string[]): Args {
     else if (v === "--cache-right") a.cacheRight = true;
     else if (v === "--no-recheck") a.recheck = false;
     else if (v === "--computed") a.computed = true;
+    else if (v === "--aria") a.aria = true;
     else if (v === "--mask-media") a.maskMedia = true;
     else if (v === "--no-freeze-video") a.freezeVideo = false;
     else if (v === "--sitemap") {
@@ -250,6 +251,8 @@ const HELP = `
                         varies per load. Off anyway under --cache-right.
     --computed          Also diff every element's computed style and ::before/::after,
                         and say whether the markup is identical. Pixels still decide.
+    --aria              Also diff each page's accessibility tree (Playwright's ARIA
+                        snapshot) line by line. Pixels still decide.
     --sitemap[=<url>]   Add the paths in the right side's sitemap (one index level, files
                         dropped). Default <right>/sitemap.xml; give a URL or path with =.
     --dry-run           Print the URL pairs and exit. No browser.

src/compare.ts

@@ -252,7 +252,7 @@ function backtrack(trace: Int32Array[], n: number, m: number, D: number) {
   return pairs.reverse();
 }
 
-function textDiff(a: string[], b: string[]): TextDiff {
+export function textDiff(a: string[], b: string[]): TextDiff {
   const pairs = myers(a, b);
   if (!pairs) {
     // Too different to align: fall back to plain membership.

src/computed.ts

@@ -1,3 +1,4 @@
+import { textDiff } from "./compare.ts";
 import { name } from "./config.ts";
 import { PROPS } from "./locate.ts";
 import type { ComparedResult, RouteResult } from "./report.ts";
@@ -34,7 +35,17 @@ export interface Finding extends StyleChange {
   at: { url: string; widths: number[] }[];
 }
 
+export interface AriaDiff {
+  /** Lines of Playwright's ARIA snapshot only the left (your change) has, at most `MAX_ARIA_LINES`; both empty: identical. */
+  added: string[];
+  removed: string[];
+  /** Lines past `MAX_ARIA_LINES` on either side, left out. */
+  truncated: number;
+}
+
 const MAX_CHANGES = 500;
+const MAX_ARIA_LINES = 200;
+const ARIA_LINES_SHOWN = 10;
 const MAX_UNMATCHED = 20;
 const FIELDS = [...PROPS, "content"];
 
@@ -66,16 +77,19 @@ const hashy = (t: string) => (t.match(/\d+|[A-Za-z]+/g)?.length ?? 0) >= 4;
 
 /** In URL attributes, bundler hashes in a file name (`main.3f9a8c1d.png`, `?v=…`) read the same on both sides. */
 export const normalizeAssets = (html: string) =>
+  html.replace(
+    / (src|href|srcset|poster)="([^"]*)"/g,
+    (_, n: string, v: string) => ` ${n}="${normalizeUrl(v)}"`,
+  );
+
+const normalizeUrl = (v: string) =>
   // ponytail: hex/base36 hashes between separators; add a framework's build-id path when one shows up
-  html.replace(/ (src|href|srcset|poster)="([^"]*)"/g, (_, n: string, v: string) => {
-    const u = v
-      .replace(
-        /([.\-_])([A-Za-z0-9]{8,})(?=(?:[.\-_][A-Za-z0-9]+)*\.[A-Za-z0-9]{2,5}(?:$|[?#\s,]))/g,
-        (m, sep: string, t: string) => (hashy(t) ? `${sep}#` : m),
-      )
-      .replace(/([?&](?:v|ver|version|hash)=)[\w.-]+/g, "$1#");
-    return ` ${n}="${u}"`;
-  });
+  v
+    .replace(
+      /([.\-_])([A-Za-z0-9]{8,})(?=(?:[.\-_][A-Za-z0-9]+)*\.[A-Za-z0-9]{2,5}(?:$|[?#\s,]))/g,
+      (m, sep: string, t: string) => (hashy(t) ? `${sep}#` : m),
+    )
+    .replace(/([?&](?:v|ver|version|hash)=)[\w.-]+/g, "$1#");
 
 /** A page's throw is its text after the first `Error: `, without the stack page.evaluate appends. */
 export const pageError = (e: unknown) => {
@@ -115,19 +129,20 @@ export function collapseInherited<C extends { key: string; prop: string; values:
   });
 }
 
-/** `from`'s origin, also protocol-relative, written as `to`'s: in keys, markup and styles. */
-function rebase(s: Snapshot, from: string, to: string): Snapshot {
+/** `from`'s origin, also protocol-relative, written as `to`'s. */
+function rebaseText(s: string, from: string, to: string): string {
   const [f, t] = [new URL(from), new URL(to)];
   if (f.origin === t.origin || f.origin === "null" || t.origin === "null") return s;
   const esc = (x: string) => x.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
   // Not `:46110` for `:4611`, nor `localhost.example.com` for `localhost`.
   const re = new RegExp(`(${esc(f.protocol)})?//${esc(f.host)}(?![\\w.:-])`, "g");
-  // Origins hold no `"` or `\`, so the JSON text rewrites like the strings it encodes.
-  return JSON.parse(
-    JSON.stringify(s).replace(re, (_, p?: string) => `${p ? t.protocol : ""}//${t.host}`),
-  ) as Snapshot;
+  return s.replace(re, (_, p?: string) => `${p ? t.protocol : ""}//${t.host}`);
 }
 
+/** In keys, markup and styles. Origins hold no `"` or `\`, so the JSON text rewrites like the strings it encodes. */
+const rebase = (s: Snapshot, from: string, to: string): Snapshot =>
+  JSON.parse(rebaseText(JSON.stringify(s), from, to)) as Snapshot;
+
 export function diffSnapshots(
   left: Snapshot,
   right: Snapshot,
@@ -279,3 +294,62 @@ export function fmtComputed(results: RouteResult[], routes: string[]): string[]
     out.push(`  (+${unmatched.length - UNMATCHED_SHOWN} more routes with unmatched elements)`);
   return out;
 }
+
+// ponytail: raw indented lines, so a wrapper role added or removed re-indents everything below it; key lines by role path if that gets noisy
+export function diffAria(left: string, right: string, origins?: [string, string]): AriaDiff {
+  const lines = (t: string) =>
+    t.split("\n").map((l) => l.replace(/^(\s*- \/url: )(.*)$/, (_, p, u) => p + normalizeUrl(u)));
+  const { added, removed } = textDiff(
+    lines(origins ? rebaseText(left, ...origins) : left),
+    lines(right),
+  );
+  return {
+    added: added.slice(0, MAX_ARIA_LINES),
+    removed: removed.slice(0, MAX_ARIA_LINES),
+    truncated:
+      Math.max(0, added.length - MAX_ARIA_LINES) + Math.max(0, removed.length - MAX_ARIA_LINES),
+  };
+}
+
+/** The console block: identical routes counted, then each distinct diff once with the widths it was seen at. */
+export function fmtAria(results: RouteResult[], routes: string[]): string[] {
+  const rows = results.filter((r): r is ComparedResult => r.status === "compared" && "aria" in r);
+  if (!rows.length) return [];
+  const groups = new Map<string, { at: string; widths: number[]; lines: string[]; more: number }>();
+  const failed: string[] = [];
+  const differs: string[] = [];
+  let identical = 0;
+  for (const route of routes) {
+    const rs = rows.filter((r) => name(r) === route);
+    if (!rs.length) continue;
+    const bad = rs.find((r) => !r.aria);
+    if (bad) failed.push(`${route}@${bad.width} (${bad.ariaError ?? "unknown"})`);
+    const diffs = rs.filter((r) => r.aria && (r.aria.added.length || r.aria.removed.length));
+    if (diffs.length) differs.push(route);
+    else if (rs.some((r) => r.aria)) identical++;
+    for (const r of diffs) {
+      const { added, removed, truncated } = r.aria!;
+      const id = JSON.stringify([route, added, removed]);
+      const g = groups.get(id);
+      if (g) g.widths.push(r.width);
+      else {
+        const lines = [...added.map((l) => `+${l}`), ...removed.map((l) => `-${l}`)];
+        groups.set(id, { at: route, widths: [r.width], lines, more: truncated });
+      }
+    }
+  }
+  const out = [
+    `aria: identical on ${identical} of ${identical + differs.length} route(s)` +
+      (differs.length ? `, differs ${differs.join(" ")}` : "") +
+      (failed.length ? `; snapshot failed on ${failed.join(" ")}` : ""),
+  ];
+  const all = [...groups.values()];
+  for (const g of all.slice(0, SHOWN)) {
+    out.push(`  ${g.at}@${g.widths.join(",")}`);
+    out.push(...g.lines.slice(0, ARIA_LINES_SHOWN).map((l) => `    ${l}`));
+    const more = Math.max(0, g.lines.length - ARIA_LINES_SHOWN) + g.more;
+    if (more) out.push(`    (+${more} more in results.json)`);
+  }
+  if (all.length > SHOWN) out.push(`  (+${all.length - SHOWN} more in results.json)`);
+  return out;
+}

src/config.ts

@@ -292,6 +292,8 @@ export interface RenderdiffConfig {
   masks?: Mask[];
   /** Also diff every element's computed style, and say whether the markup is identical. */
   computed?: boolean;
+  /** Also diff each page's ARIA snapshot. */
+  aria?: boolean;
   /** Mask every `iframe` on every route. */
   maskMedia?: boolean;
   /** Keep every video paused on frame 0 (its poster if it never loads) and hide video.js overlays. Default true. */
@@ -320,6 +322,8 @@ export interface ResolvedConfig {
   freezeVideo: boolean;
   /** Optional, not null, so an unset value leaves the config hash unchanged. */
   lottieFrame?: number;
+  /** Optional like `lottieFrame`. */
+  aria?: true;
 }
 
 /** package.json's version; the same relative path resolves from `src/` and `dist/`. */
@@ -492,6 +496,7 @@ function validate(c: Record<string, unknown>, file: string) {
     }
   }
   if (c.computed !== undefined && typeof c.computed !== "boolean") bad("computed", "a boolean");
+  if (c.aria !== undefined && typeof c.aria !== "boolean") bad("aria", "a boolean");
   if (c.threshold !== undefined) {
     const t = c.threshold;
     if (typeof t !== "number" || !Number.isFinite(t) || t < 0 || t > 1) {
@@ -548,6 +553,7 @@ export interface CliOverrides {
   /** Per side, from --storage-state / --left-storage-state / --right-storage-state. */
   storageState?: { left?: string; right?: string };
   computed?: boolean;
+  aria?: boolean;
   /** From --mask, appended to the config's masks. */
   masks?: string[];
   maskMedia?: boolean;
@@ -609,6 +615,7 @@ export function resolveConfig(file: RenderdiffConfig, cli: CliOverrides): Resolv
     computed: cli.computed ?? file.computed ?? false,
     freezeVideo,
     lottieFrame: file.lottieFrame,
+    aria: (cli.aria ?? file.aria) || undefined,
   };
 }
 

src/index.ts

@@ -38,5 +38,5 @@ export {
 
 export { type Band, type Box, type Region, type TextDiff } from "./compare.ts";
 export { type ElementDiff } from "./locate.ts";
-export { type ComputedDiff, type Finding, type StyleChange } from "./computed.ts";
+export { type AriaDiff, type ComputedDiff, type Finding, type StyleChange } from "./computed.ts";
 export { type Accepted, type Fingerprint } from "./accept.ts";