From 860d4af98c91bd3e981cc4cae8b1471e91288e3a Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:29:58 +0200 Subject: [PATCH 1/2] Add --cache-right: replay the reference's network from a HAR The right page still loads, settles and is shot live, so element attribution, blank/failed-asset refusals and unstable detection are unchanged; only round trips to the reference are skipped. Opt-in, a recording is replayed for an hour and only when it covers every shot in the run. The run prints a right: line and records meta.rightCache. Co-Authored-By: Claude Opus 5.5 --- README.md | 15 ++++++- docs/how-it-works.md | 9 ++++ skills/renderdiff/SKILL.md | 6 ++- skills/renderdiff/reference.md | 8 +++- src/cache.ts | 63 ++++++++++++++++++++++++++ src/cli.ts | 7 +++ src/report.ts | 6 +++ src/run.ts | 59 ++++++++++++++++++++++--- tests/unit/cache.test.ts | 81 ++++++++++++++++++++++++++++++++++ tests/visual/accepted.test.ts | 33 ++++++++++++++ 10 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 src/cache.ts create mode 100644 tests/unit/cache.test.ts diff --git a/README.md b/README.md index 4fd266a..0c1191d 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ renderdiff [options] | `--left-storage-state `, `--right-storage-state ` | That side's storage state, over `--storage-state`. One side alone is fine. | | `--accept ` | Record this run's differences as accepted, with a reason. See **Accepted differences and reruns**. | | `--rerun-failed` | Rerun only what the last run in `--out` didn't find clean or accepted, keeping the rest. | +| `--cache-right` | Replay the right side's network traffic from the last `--cache-right` run in `--out`, when it's under an hour old and covers every shot; otherwise load it live and record it. See **Accepted differences and reruns**. | | `--dry-run` | Print the URL pairs and exit. No browser. | | `--open` | Open the report when the run finishes. | | `--progress` | A line per shot as it finishes, then the worst scores. The default on a terminal; piped, renderdiff prints one compact summary at the end. | @@ -188,6 +189,7 @@ only. | `ERROR …` | Unmeasured, which is neither a pass nor a fail. Exits 3. | | `accepted:` / `stale accepted:` | Shots whose every difference was accepted, and accepted entries the run no longer saw. See below. | | `rerun:` | With `--rerun-failed`: how many shots were rerun and how many reused. | +| `right:` | With `--cache-right`: whether the right side was replayed, and from when, or loaded live and recorded. | Read text first, then `dom`, then the crops for anything `dom` doesn't explain. Full detail, including the `--progress` and report forms: [docs/output.md](https://github.com/tmashara/renderdiff/blob/main/docs/output.md). @@ -211,6 +213,16 @@ anything new on the same page still shows. `--rerun-failed` reruns only what the the previous one. Matching rules, what can still hide, and stale entries: [docs/accepted.md](https://github.com/tmashara/renderdiff/blob/main/docs/accepted.md). +While iterating against a slow or remote reference, add `--cache-right`. The first run loads +the right side live and records its network traffic as a HAR in `/.cache/` (kept when the out dir is wiped); later runs +replay it instead of fetching from the reference, for up to an hour, as long as it covers every +shot in the run. The right page still renders, settles and gets shot fresh, so element names, +blank-page and failed-asset checks and unstable pixels all work as before. Only the round trips +are skipped. A request the recording doesn't have goes to the network. A replay can't see a +deploy to the reference, so the run prints a `right: replayed from …` line, and +`meta.rightCache` in `results.json` records it. Drop the flag for the final run. Recordings +of a signed-in right side hold that session's responses, so keep the out dir out of git. + ## Output Everything lands in the out dir (`./renderdiff-out` by default): @@ -235,7 +247,8 @@ Everything lands in the out dir (`./renderdiff-out` by default): "blockUrlPatterns": ["…"], "hideSelectors": ["…"], "masks": [{ "urlPath": "/pricing", "reason": "live star count" }], // no urlPath: every route "browser": { "name": "chromium", "version": "…" }, // "chrome" when it fell back - "storageState": { "left": "auth.json", "right": null } // file names only + "storageState": { "left": "auth.json", "right": null }, // file names only + "rightCache": null // --cache-right: { "replayedFrom": "…" }, or { "replayedFrom": null } when recorded }, "results": [ { diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 27da5d9..fb42ec7 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -37,6 +37,15 @@ honestly. This is what renderdiff does to make sure of that. cross (a storage state file seeds each context on its own). Both sides of a route load at the same time. - **Retries once** per shot; a live site can time out on a cold edge. +- **Replays, never reuses, a cached reference.** With `--cache-right`, the right side's network + traffic comes from a HAR recorded by an earlier run, but the page is still loaded, settled and + shot under this run's clock. Element attribution needs the live document anyway, so nothing + measured is reused. A recording is used only while it's under an hour old, covers every shot in + the run, and was made with the same right origin, right storage state, blocked URLs, pinned + context options, renderdiff version and browser. Otherwise the right side loads live and is + recorded again. A request the recording lacks goes to the network. Shots that errored aren't + recorded, so an error is never replayed. The run says when it replayed (`right:` line, + `meta.rightCache`), because a recording can't see a deploy to the reference. - **Stitches tall pages.** A single screenshot past 16,384px can come out truncated or wrapped (Chrome repeats the top of the page; recent headless Chromium does not), so taller pages, and element shots taller than that, are captured as tiles through CDP and stitched diff --git a/skills/renderdiff/SKILL.md b/skills/renderdiff/SKILL.md index beade8e..b51dec5 100644 --- a/skills/renderdiff/SKILL.md +++ b/skills/renderdiff/SKILL.md @@ -39,8 +39,10 @@ legitimate: [reference.md](reference.md). 7. `Read` the crops under each row (next to results.json) for what `dom` doesn't explain. Left is on top or left; red box = change, red seam = missing rows. 8. Fix or justify every band, region and text line. After each migration step, - `--rerun-failed` (same arguments) reruns only what isn't clean or accepted; finish with - all routes at default widths. For a multi-width row, open the widest and narrowest crop. + `--rerun-failed` (same arguments) reruns only what isn't clean or accepted; add + `--cache-right` to replay the reference's traffic from the last such run (an hour at most) + instead of fetching it again. Finish with all routes at default widths, without + `--cache-right`. For a multi-width row, open the widest and narrowest crop. `accepted:` routes are known; a row saying `1 accepted` still has something new. Exit: 0 nothing below `--threshold` (**without it, a diff still exits 0**) · 1 below diff --git a/skills/renderdiff/reference.md b/skills/renderdiff/reference.md index 6e43570..6fd1af8 100644 --- a/skills/renderdiff/reference.md +++ b/skills/renderdiff/reference.md @@ -59,7 +59,8 @@ insertion point, `layout` `stacked` = left on top or `side` = left on the left, Paths are relative to the out dir. `summary` has `compared`, `errors`, `mean`, `worst` and `belowThreshold`; `meta` records origins, widths, threshold, the pinned clock, block patterns, hidden selectors, mask reasons, the browser (`chromium`, or `chrome` when it fell -back to the system Chrome) and each side's storage state file name. `hook: true` marks a +back to the system Chrome), each side's storage state file name, and `rightCache` (with +`--cache-right`: `replayedFrom`, the recording's time, or null when it was recorded). `hook: true` marks a route a `before` hook ran on. The normal loop doesn't need it. ## Config file @@ -105,6 +106,11 @@ cookie script ran wedges the renderer). - **`--rerun-failed`** keeps the out dir and reruns only what wasn't clean or accepted. Give it the same origins, routes, widths and config (hooks included) as the run it continues, or it exits 2. +- **`--cache-right`** replays the right side's recorded network traffic (`/.cache/`, kept by the wipe) + for up to an hour, and only if the recording covers every shot in the run. The page still + renders live, so `dom` and the refusals still hold, but a deploy to the reference goes unseen + until the recording expires. A `right: replayed from …` line says so. Never give a verdict + from a replayed run alone. - **The out dir is wiped at the start of every run** (except `--rerun-failed`). To keep a previous run's evidence next to a new one, pass `--out renderdiff-before` / `--out renderdiff-after`. A directory holding files renderdiff didn't write is refused (exit 2). diff --git a/src/cache.ts b/src/cache.ts new file mode 100644 index 0000000..bf75a63 --- /dev/null +++ b/src/cache.ts @@ -0,0 +1,63 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { CONTEXT_OPTIONS, version, type ResolvedConfig, type Route } from "./config.ts"; + +/** + * `--cache-right`: the right side's network traffic, recorded as a HAR and replayed on later + * runs. The page still loads, settles and is shot live, so attribution and every refusal + * work as on a live load; only the round trips to the reference are skipped. + */ + +/** In the out dir, kept when it is wiped. */ +export const CACHE_DIR = ".cache"; + +/** A recording older than this is recorded again, so prod can't drift unseen for long. */ +export const REPLAY_TTL = 60 * 60_000; + +/** A shot's identity in a recording: what decides which requests its load makes. */ +export const shotId = (route: Route, width: number) => + JSON.stringify([route.right, width, route.before && String(route.before)]); + +/** Everything that changes what the right origin serves, so one recording never answers another. */ +export function cacheFile(cfg: ResolvedConfig, browser: { name: string; version: string }): string { + const state = cfg.storageState.right && fs.readFileSync(cfg.storageState.right, "utf8"); + const key = createHash("sha256") + .update( + JSON.stringify([version(), browser, cfg.right, state, CONTEXT_OPTIONS, cfg.blockUrlPatterns]), + ) + .digest("hex") + .slice(0, 16); + return path.join(cfg.outDir, CACHE_DIR, `right-${key}.har`); +} + +interface Har { + log: { entries: unknown[]; _renderdiff?: { recordedAt: number; shots: string[] } }; +} + +/** When the recording was made, if it covers every shot and is under `REPLAY_TTL` old; else null. */ +export function replayable(file: string, shots: string[], now = Date.now()): number | null { + let rec; + try { + rec = (JSON.parse(fs.readFileSync(file, "utf8")) as Har).log._renderdiff; + } catch { + return null; + } + if (!rec || !(now - rec.recordedAt >= 0 && now - rec.recordedAt < REPLAY_TTL)) return null; + return shots.every((s) => rec.shots.includes(s)) ? rec.recordedAt : null; +} + +/** Merge each worker's recording into `file`, for `shots` only; the parts are removed. */ +export function saveRecording(file: string, parts: string[], shots: string[], recordedAt: number) { + const hars = parts + .filter((p) => fs.existsSync(p)) + .map((p) => JSON.parse(fs.readFileSync(p, "utf8")) as Har); + for (const p of parts) fs.rmSync(p, { force: true }); + if (!hars.length || !shots.length) return; + const har = hars[0]!; + har.log.entries = hars.flatMap((h) => h.log.entries); + har.log._renderdiff = { recordedAt, shots }; + // Renamed into place, so a run killed mid-write never leaves a half recording to replay. + fs.writeFileSync(`${file}.tmp`, JSON.stringify(har)); + fs.renameSync(`${file}.tmp`, file); +} diff --git a/src/cli.ts b/src/cli.ts index a0c7432..c7d5765 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -39,6 +39,7 @@ interface Args extends CliOverrides { /** The reason to accept this run's differences with. */ accept?: string; rerunFailed: boolean; + cacheRight: boolean; } /** `Number(raw)`, but reject anything that isn't a finite number with a flag-named error. */ @@ -55,6 +56,7 @@ function parseArgs(argv: string[]): Args { dryRun: false, open: false, rerunFailed: false, + cacheRight: false, help: false, version: false, routes: [], @@ -86,6 +88,7 @@ function parseArgs(argv: string[]): Args { else if (v === "--progress") a.progress = true; else if (v === "--no-progress") a.progress = false; else if (v === "--rerun-failed") a.rerunFailed = true; + else if (v === "--cache-right") a.cacheRight = true; else if (v === "--accept") { a.accept = next().trim(); if (!a.accept) throw new Error("--accept needs a reason"); @@ -199,6 +202,9 @@ const HELP = ` --accept Accept this run's differences, scoped by its routes and widths. --rerun-failed Rerun what the last run in --out didn't find clean or accepted, keeping the rest. Same origins, routes and widths as that run. + --cache-right Replay the right side's network from the last --cache-right run + in --out, if under an hour old and covering every shot; else load + it live and record it. The page still renders and is shot fresh. --dry-run Print the URL pairs and exit. No browser. --open Open the report when the run finishes. --progress A line per shot as it finishes. The default on a terminal; @@ -326,6 +332,7 @@ try { progress: args.progress ?? !!process.stdout.isTTY, accept: args.accept ?? null, rerun: args.rerunFailed, + cacheRight: args.cacheRight, acceptedFile: path.join(configPath ? path.dirname(configPath) : process.cwd(), ACCEPTED_FILE), }), ); diff --git a/src/report.ts b/src/report.ts index 8d61468..9e89150 100644 --- a/src/report.ts +++ b/src/report.ts @@ -94,6 +94,11 @@ export interface RunMeta { storageState: { left: string | null; right: string | null }; /** A hash of everything that decides how a shot is taken; `--rerun-failed` needs it unchanged. */ config: string; + /** + * With `--cache-right`: when the recording the right side's network was replayed from was + * made, ISO, or null when it loaded live and was recorded. Null without the flag. + */ + rightCache: { replayedFrom: string | null } | null; } export interface Summary { @@ -333,6 +338,7 @@ export async function writeReport( .filter((s) => meta.storageState[s]) .map((s) => ` · ${s} storage state ${esc(meta.storageState[s]!)}`) .join("")} + ${meta.rightCache?.replayedFrom ? `· right replayed from a recording of ${esc(meta.rightCache.replayedFrom)}` : ""}

${section(compared)} diff --git a/src/run.ts b/src/run.ts index 3c48fd6..49ccf9c 100644 --- a/src/run.ts +++ b/src/run.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { chromium, errors, type Browser, type Page } from "playwright"; import { applyAccepted, readAccepted, updateAccepted } from "./accept.ts"; +import { CACHE_DIR, cacheFile, replayable, REPLAY_TTL, saveRecording, shotId } from "./cache.ts"; import { cancelUnstable, capture, HookError, prepare, type Shot } from "./capture.ts"; import { compare, crops, type Band } from "./compare.ts"; import { locate, type ElementDiff } from "./locate.ts"; @@ -51,6 +52,8 @@ interface RunOpts { /** Rerun only what the previous results.json in the out dir didn't find clean or accepted. */ rerun: boolean; acceptedFile: string; + /** Replay the right side's traffic from the out dir's recording, or record one. */ + cacheRight: boolean; } const slug = (url: string) => @@ -89,6 +92,7 @@ function slugs(routes: Route[]): Map { const ours = (f: string) => f === "report.html" || f === "results.json" || + f === CACHE_DIR || /^[a-z0-9_~@-]+\.[\d.]+\.(left|right|diff|c\d+)\.png$/.test(f); function wipeable(dir: string): boolean { @@ -158,6 +162,7 @@ export async function run({ accept, rerun, acceptedFile, + cacheRight, }: RunOpts): Promise { const names = routes.map(name); const setup = { @@ -201,7 +206,11 @@ export async function run({ ); return 2; } - if (!prev) fs.rmSync(cfg.outDir, { recursive: true, force: true }); + if (!prev && fs.existsSync(cfg.outDir)) { + for (const f of fs.readdirSync(cfg.outDir)) { + if (f !== CACHE_DIR) fs.rmSync(path.join(cfg.outDir, f), { recursive: true, force: true }); + } + } fs.mkdirSync(cfg.outDir, { recursive: true }); if (prev) applyAccepted(prev.results, accepted); @@ -222,8 +231,18 @@ export async function run({ let done = 0; // Whole minute, so an HH:MM display reads the same on both sides. const now = Math.floor(Date.now() / 60_000) * 60_000; + const har = launched && cacheRight ? cacheFile(cfg, browserOf(launched)) : null; + const replayedAt = + har && + replayable( + har, + jobs.map((j) => shotId(j.route, j.width)), + ); + const recorded: string[] = []; + if (har && !replayedAt) fs.mkdirSync(path.dirname(har), { recursive: true }); + const part = (i: number) => `${har}.${i}.part`; - const worker = async (browser: Browser) => { + const worker = async (browser: Browser, i: number) => { // One context per origin so cookies/localStorage never cross between them. const leftCtx = await browser.newContext({ ...CONTEXT_OPTIONS, @@ -233,6 +252,14 @@ export async function run({ ...CONTEXT_OPTIONS, storageState: cfg.storageState.right ?? undefined, }); + if (har) { + await rightCtx.routeFromHAR( + replayedAt ? har : part(i), + replayedAt + ? { notFound: "fallback" } + : { update: true, updateContent: "embed", updateMode: "minimal" }, + ); + } const leftPage = await leftCtx.newPage(); const rightPage = await rightCtx.newPage(); await prepare(leftPage, cfg, now); @@ -301,6 +328,7 @@ export async function run({ ...made.map((c, i) => write(cropped[i]!.path, c.png)), ]); + recorded.push(shotId(route, width)); results.push({ ...entry, status: "compared", @@ -356,12 +384,16 @@ export async function run({ console.log(`Capturing ${jobs.length} pair(s) with ${cfg.concurrency} worker(s)...\n`); } if (launched) { - await Promise.all(Array.from({ length: cfg.concurrency }, () => worker(launched.browser))); + await Promise.all( + Array.from({ length: cfg.concurrency }, (_, i) => worker(launched.browser, i)), + ); await launched.browser.close(); } - const browserMeta = launched - ? { name: launched.channel ?? "chromium", version: launched.browser.version() } - : prev!.meta.browser; + const browserMeta = launched ? browserOf(launched) : prev!.meta.browser; + if (har && !replayedAt) { + const parts = Array.from({ length: cfg.concurrency }, (_, i) => part(i)); + saveRecording(har, parts, recorded, now); + } // A rerun overwrites a shot's files in place; drop the ones it no longer writes, ours only. const kept = new Set(results.flatMap(files)); @@ -405,6 +437,9 @@ export async function run({ ...rest, browser: browserMeta, config, + rightCache: har + ? { replayedFrom: replayedAt ? new Date(replayedAt).toISOString() : null } + : null, }; const reportPath = await writeReport(results, meta, cfg, stale); const resultsPath = writeResults(results, meta, cfg.outDir, stale); @@ -416,6 +451,13 @@ export async function run({ `${results.length - jobs.length} reused, ${names.length - rerunRoutes} route(s) untouched`, ); } + if (har) { + console.log( + replayedAt + ? `right: replayed from a recording of ${new Date(replayedAt).toISOString()} (--cache-right)` + : `right: loaded live and recorded for --cache-right (replayed for ${REPLAY_TTL / 60_000} min)`, + ); + } if (stale.length) { console.log( `stale accepted: ${stale.map((a) => `${a.route}@${a.width}`).join(" ")} — ${acceptedFile}`, @@ -440,6 +482,11 @@ export async function run({ return below.length ? 1 : results.some((r) => r.status === "error") ? 3 : 0; } +const browserOf = (l: { browser: Browser; channel: string | null }) => ({ + name: l.channel ?? "chromium", + version: l.browser.version(), +}); + /** * A missing browser binary is the likeliest first-run failure, so fall back to the system * Chrome (both sides share it, so the comparison stays symmetric), then name the fix. diff --git a/tests/unit/cache.test.ts b/tests/unit/cache.test.ts new file mode 100644 index 0000000..73a3f6c --- /dev/null +++ b/tests/unit/cache.test.ts @@ -0,0 +1,81 @@ +/** + * `--cache-right`: when a recording of the right side may be replayed, and what keys it. No + * browser: the recordings are hand-written HARs. + */ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { after, describe, test } from "node:test"; +import { cacheFile, replayable, REPLAY_TTL, saveRecording, shotId } from "../../src/cache.ts"; +import { parseRoutes, resolveConfig } from "../../src/config.ts"; +import { cleanupTempDirs, tempDir } from "../helpers/cli.ts"; + +after(cleanupTempDirs); + +const browser = { name: "chromium", version: "1" }; +const cfg = (over: Parameters[1] = {}) => + resolveConfig({}, { left: "http://l.test", right: "http://r.test", outDir: "/tmp/o", ...over }); +const [route] = parseRoutes(["/a"]); + +function record(shots: string[], recordedAt: number) { + const dir = tempDir(); + const parts = [0, 1].map((i) => path.join(dir, `${i}.part`)); + parts.forEach((p, i) => + fs.writeFileSync(p, JSON.stringify({ log: { version: "1.2", entries: [{ i }] } })), + ); + const file = path.join(dir, "right.har"); + saveRecording(file, [...parts, path.join(dir, "missing.part")], shots, recordedAt); + return { file, parts }; +} + +describe("replayable", () => { + const now = Date.now(); + const a = shotId(route!, 1440); + const b = shotId(route!, 375); + + test("a fresh recording covering every shot replays; the parts merge and go", () => { + const { file, parts } = record([a, b], now - 1000); + assert.equal(replayable(file, [a], now), now - 1000); + assert.deepEqual(JSON.parse(fs.readFileSync(file, "utf8")).log.entries, [{ i: 0 }, { i: 1 }]); + assert.ok(parts.every((p) => !fs.existsSync(p))); + }); + + test("a shot it lacks, an expired or future recording, or none at all loads live", () => { + assert.equal(replayable(record([a], now).file, [a, b], now), null); + assert.equal(replayable(record([a], now - REPLAY_TTL).file, [a], now), null); + assert.equal(replayable(record([a], now + 60_000).file, [a], now), null); + assert.equal(replayable(path.join(tempDir(), "none.har"), [a], now), null); + }); + + test("a run where no shot succeeded writes nothing", () => { + assert.equal(fs.existsSync(record([], now).file), false); + }); +}); + +describe("keys", () => { + test("a hook, the right path or the width is a different shot", () => { + const hooked = parseRoutes(["/a"], undefined, async () => {})[0]!; + const paired = parseRoutes(["/b::/a"])[0]!; + assert.notEqual(shotId(route!, 1440), shotId(hooked, 1440)); + assert.notEqual(shotId(route!, 1440), shotId(route!, 375)); + assert.equal(shotId(route!, 1440), shotId(paired, 1440), "the left path doesn't matter"); + }); + + test("the right origin, its storage state and the browser change the file", () => { + const dir = tempDir(); + const state = path.join(dir, "auth.json"); + fs.writeFileSync(state, JSON.stringify({ cookies: [], origins: [] })); + const signed = cacheFile(cfg({ storageState: { right: state } }), browser); + fs.writeFileSync(state, JSON.stringify({ cookies: [{ name: "s" }], origins: [] })); + const files = [ + cacheFile(cfg(), browser), + cacheFile(cfg({ right: "http://other.test" }), browser), + cacheFile(cfg(), { ...browser, version: "2" }), + signed, + cacheFile(cfg({ storageState: { right: state } }), browser), + ]; + assert.equal(new Set(files).size, files.length); + assert.equal(cacheFile(cfg({ left: "http://x.test" }), browser), files[0]); + assert.equal(path.dirname(files[0]!), "/tmp/o/.cache"); + }); +}); diff --git a/tests/visual/accepted.test.ts b/tests/visual/accepted.test.ts index 03e967b..7618b94 100644 --- a/tests/visual/accepted.test.ts +++ b/tests/visual/accepted.test.ts @@ -246,3 +246,36 @@ describe("--rerun-failed", () => { assert.match(fresh.stderr, /no .*results\.json to rerun from/); }); }); + +describe("--cache-right", () => { + test("replays the right side's traffic on a repeat run, with the same results", async () => { + const routes = ["/nav", "/color"]; + const first = await runCli(args(routes, ["--cache-right"])); + assert.equal(first.code, 0, context(first)); + assert.match(first.stdout, /^right: loaded live and recorded for --cache-right/m); + const was = results(first.outDir); + assert.equal(was.meta.rightCache?.replayedFrom, null); + + const hits = site.hits("/nav"); + const again = await runCli(args(routes, ["--cache-right"]), { outDir: first.outDir }); + assert.equal(again.code, 0, context(again)); + assert.equal(site.hits("/nav"), hits + 1, "only the left side is loaded"); + assert.match(again.stdout, /^right: replayed from a recording of \S+ \(--cache-right\)$/m); + const now = results(again.outDir); + assert.ok(now.meta.rightCache?.replayedFrom, context(again)); + for (const route of routes) { + assert.deepEqual(result(now, route, 1440), result(was, route, 1440)); + } + assert.ok( + result(now, "/nav", 1440).bands?.some((b) => b.elements?.[0]?.side === "right"), + "elements are still named on a replayed right side", + ); + + const wider = await runCli(args([...routes, "/"], ["--cache-right"]), { + outDir: first.outDir, + }); + assert.equal(wider.code, 0, context(wider)); + assert.equal(site.hits("/nav"), hits + 3, "a shot the recording lacks loads it all live"); + assert.match(wider.stdout, /^right: loaded live and recorded/m); + }); +}); From 7563a78fa2c2f61c5e6a0c9a22b4f9bbbd866b19 Mon Sep 17 00:00:00 2001 From: Timur Mashara <62305878+tmashara@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:40:59 +0200 Subject: [PATCH 2/2] Keep replayed shots and errors out of later runs A shot measured against a replay is marked rightReplayedFrom and reused by --rerun-failed only in a run that would replay it; meta carries the oldest. Recordings drop error responses and repeats, skip shots with failed assets, replace older recordings, keep their index in a sidecar, and a failed save no longer loses the report. .cache survives the wipe only with --cache-right. Co-Authored-By: Claude Opus 5.5 --- README.md | 19 ++++---- docs/how-it-works.md | 9 ++-- skills/renderdiff/reference.md | 7 +-- src/cache.ts | 61 +++++++++++++++---------- src/report.ts | 11 +++-- src/run.ts | 39 +++++++++++----- tests/unit/cache.test.ts | 46 +++++++++++++++---- tests/visual/accepted.test.ts | 81 ++++++++++++++++++++++++++++++++-- 8 files changed, 203 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 0c1191d..dcf331b 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ renderdiff [options] | `--left-storage-state `, `--right-storage-state ` | That side's storage state, over `--storage-state`. One side alone is fine. | | `--accept ` | Record this run's differences as accepted, with a reason. See **Accepted differences and reruns**. | | `--rerun-failed` | Rerun only what the last run in `--out` didn't find clean or accepted, keeping the rest. | -| `--cache-right` | Replay the right side's network traffic from the last `--cache-right` run in `--out`, when it's under an hour old and covers every shot; otherwise load it live and record it. See **Accepted differences and reruns**. | +| `--cache-right` | Replay the right side's network traffic from the last `--cache-right` run in `--out`, when it's under an hour old and covers every shot; otherwise load it live and record it. Keeps `/.cache/` when the out dir is wiped. See **Accepted differences and reruns**. | | `--dry-run` | Print the URL pairs and exit. No browser. | | `--open` | Open the report when the run finishes. | | `--progress` | A line per shot as it finishes, then the worst scores. The default on a terminal; piped, renderdiff prints one compact summary at the end. | @@ -213,15 +213,12 @@ anything new on the same page still shows. `--rerun-failed` reruns only what the the previous one. Matching rules, what can still hide, and stale entries: [docs/accepted.md](https://github.com/tmashara/renderdiff/blob/main/docs/accepted.md). -While iterating against a slow or remote reference, add `--cache-right`. The first run loads -the right side live and records its network traffic as a HAR in `/.cache/` (kept when the out dir is wiped); later runs -replay it instead of fetching from the reference, for up to an hour, as long as it covers every -shot in the run. The right page still renders, settles and gets shot fresh, so element names, -blank-page and failed-asset checks and unstable pixels all work as before. Only the round trips -are skipped. A request the recording doesn't have goes to the network. A replay can't see a -deploy to the reference, so the run prints a `right: replayed from …` line, and -`meta.rightCache` in `results.json` records it. Drop the flag for the final run. Recordings -of a signed-in right side hold that session's responses, so keep the out dir out of git. +Against a slow reference, add `--cache-right` while iterating: it records the right side's +network traffic in `/.cache/` and replays it for up to an hour. The page still renders +and is shot fresh, so only the round trips are skipped. A replay can't see a deploy, so the run +says `right: replayed from …`; drop the flag for the final run. With `--rerun-failed`, a new +recording covers only the rerun shots. It holds a signed-in session's responses: keep the out +dir out of git. ## Output @@ -248,7 +245,7 @@ Everything lands in the out dir (`./renderdiff-out` by default): "masks": [{ "urlPath": "/pricing", "reason": "live star count" }], // no urlPath: every route "browser": { "name": "chromium", "version": "…" }, // "chrome" when it fell back "storageState": { "left": "auth.json", "right": null }, // file names only - "rightCache": null // --cache-right: { "replayedFrom": "…" }, or { "replayedFrom": null } when recorded + "rightReplayedFrom": null // --cache-right: when the replayed recording was made }, "results": [ { diff --git a/docs/how-it-works.md b/docs/how-it-works.md index fb42ec7..342559a 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -43,9 +43,12 @@ honestly. This is what renderdiff does to make sure of that. measured is reused. A recording is used only while it's under an hour old, covers every shot in the run, and was made with the same right origin, right storage state, blocked URLs, pinned context options, renderdiff version and browser. Otherwise the right side loads live and is - recorded again. A request the recording lacks goes to the network. Shots that errored aren't - recorded, so an error is never replayed. The run says when it replayed (`right:` line, - `meta.rightCache`), because a recording can't see a deploy to the reference. + recorded again, replacing any older recording; with `--rerun-failed` it covers only the + rerun shots. A request the recording lacks goes to the network. Error responses aren't + recorded, nor are shots that errored or failed an asset, so an error is never replayed. A shot + measured against a replay is marked `rightReplayedFrom`, and `--rerun-failed` reuses it only + in a run that would replay it too. The run says when it replayed (`right:` line, + `meta.rightReplayedFrom`), because a recording can't see a deploy to the reference. - **Stitches tall pages.** A single screenshot past 16,384px can come out truncated or wrapped (Chrome repeats the top of the page; recent headless Chromium does not), so taller pages, and element shots taller than that, are captured as tiles through CDP and stitched diff --git a/skills/renderdiff/reference.md b/skills/renderdiff/reference.md index 6fd1af8..cc0fe12 100644 --- a/skills/renderdiff/reference.md +++ b/skills/renderdiff/reference.md @@ -59,8 +59,8 @@ insertion point, `layout` `stacked` = left on top or `side` = left on the left, Paths are relative to the out dir. `summary` has `compared`, `errors`, `mean`, `worst` and `belowThreshold`; `meta` records origins, widths, threshold, the pinned clock, block patterns, hidden selectors, mask reasons, the browser (`chromium`, or `chrome` when it fell -back to the system Chrome), each side's storage state file name, and `rightCache` (with -`--cache-right`: `replayedFrom`, the recording's time, or null when it was recorded). `hook: true` marks a +back to the system Chrome), each side's storage state file name, and `rightReplayedFrom` +(with `--cache-right`, when the replayed recording was made; also on each replayed result). `hook: true` marks a route a `before` hook ran on. The normal loop doesn't need it. ## Config file @@ -110,7 +110,8 @@ cookie script ran wedges the renderer). for up to an hour, and only if the recording covers every shot in the run. The page still renders live, so `dom` and the refusals still hold, but a deploy to the reference goes unseen until the recording expires. A `right: replayed from …` line says so. Never give a verdict - from a replayed run alone. + from a replayed run alone. A `--rerun-failed --cache-right` re-record covers only the rerun + shots. - **The out dir is wiped at the start of every run** (except `--rerun-failed`). To keep a previous run's evidence next to a new one, pass `--out renderdiff-before` / `--out renderdiff-after`. A directory holding files renderdiff didn't write is refused (exit 2). diff --git a/src/cache.ts b/src/cache.ts index bf75a63..c763379 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -3,13 +3,6 @@ import fs from "node:fs"; import path from "node:path"; import { CONTEXT_OPTIONS, version, type ResolvedConfig, type Route } from "./config.ts"; -/** - * `--cache-right`: the right side's network traffic, recorded as a HAR and replayed on later - * runs. The page still loads, settles and is shot live, so attribution and every refusal - * work as on a live load; only the round trips to the reference are skipped. - */ - -/** In the out dir, kept when it is wiped. */ export const CACHE_DIR = ".cache"; /** A recording older than this is recorded again, so prod can't drift unseen for long. */ @@ -31,33 +24,53 @@ export function cacheFile(cfg: ResolvedConfig, browser: { name: string; version: return path.join(cfg.outDir, CACHE_DIR, `right-${key}.har`); } -interface Har { - log: { entries: unknown[]; _renderdiff?: { recordedAt: number; shots: string[] } }; +interface Entry { + request: { method: string; url: string }; + response: { status: number }; } /** When the recording was made, if it covers every shot and is under `REPLAY_TTL` old; else null. */ export function replayable(file: string, shots: string[], now = Date.now()): number | null { - let rec; + let rec: { recordedAt: number; shots: string[] }; try { - rec = (JSON.parse(fs.readFileSync(file, "utf8")) as Har).log._renderdiff; + rec = JSON.parse(fs.readFileSync(`${file}.json`, "utf8")); } catch { return null; } - if (!rec || !(now - rec.recordedAt >= 0 && now - rec.recordedAt < REPLAY_TTL)) return null; + if (!fs.existsSync(file) || !(now - rec.recordedAt >= 0 && now - rec.recordedAt < REPLAY_TTL)) + return null; return shots.every((s) => rec.shots.includes(s)) ? rec.recordedAt : null; } -/** Merge each worker's recording into `file`, for `shots` only; the parts are removed. */ +/** + * Merge each worker's recording into `file` for `shots`, replacing every other recording in + * its directory. The parts are removed. + */ export function saveRecording(file: string, parts: string[], shots: string[], recordedAt: number) { - const hars = parts - .filter((p) => fs.existsSync(p)) - .map((p) => JSON.parse(fs.readFileSync(p, "utf8")) as Har); - for (const p of parts) fs.rmSync(p, { force: true }); - if (!hars.length || !shots.length) return; - const har = hars[0]!; - har.log.entries = hars.flatMap((h) => h.log.entries); - har.log._renderdiff = { recordedAt, shots }; - // Renamed into place, so a run killed mid-write never leaves a half recording to replay. - fs.writeFileSync(`${file}.tmp`, JSON.stringify(har)); - fs.renameSync(`${file}.tmp`, file); + try { + const hars = parts + .filter((p) => fs.existsSync(p)) + .map((p) => JSON.parse(fs.readFileSync(p, "utf8")) as { log: { entries: Entry[] } }); + if (!hars.length || !shots.length) return; + const seen = new Set(); + const har = hars[0]!; + // Replay serves the first match, so a one-off error would answer every later load. + har.log.entries = hars + .flatMap((h) => h.log.entries) + .filter((e) => { + const key = `${e.request.method} ${e.request.url}`; + if (e.response.status >= 400 || seen.has(key)) return false; + seen.add(key); + return true; + }); + const dir = path.dirname(file); + for (const f of fs.readdirSync(dir)) { + if (f.startsWith("right-")) fs.rmSync(path.join(dir, f), { force: true }); + } + fs.writeFileSync(file, JSON.stringify(har)); + // Written last: without it the recording is never replayed. + fs.writeFileSync(`${file}.json`, JSON.stringify({ recordedAt, shots })); + } finally { + for (const p of parts) fs.rmSync(p, { force: true }); + } } diff --git a/src/report.ts b/src/report.ts index 9e89150..a5c0d0c 100644 --- a/src/report.ts +++ b/src/report.ts @@ -45,6 +45,8 @@ export interface ComparedResult extends BaseResult { accepted: boolean; /** Stylesheets and font faces that failed to load per side, e.g. `font Inter 700`. */ failed: { left: string[]; right: string[] }; + /** With `--cache-right`: when the recording the right side was replayed from was made, ISO. */ + rightReplayedFrom?: string; } /** A crop of one change: left and right panels, a 1px red box round it, or a 2px red seam. */ @@ -94,11 +96,8 @@ export interface RunMeta { storageState: { left: string | null; right: string | null }; /** A hash of everything that decides how a shot is taken; `--rerun-failed` needs it unchanged. */ config: string; - /** - * With `--cache-right`: when the recording the right side's network was replayed from was - * made, ISO, or null when it loaded live and was recorded. Null without the flag. - */ - rightCache: { replayedFrom: string | null } | null; + /** The oldest `rightReplayedFrom` among the results, or null when every right side loaded live. */ + rightReplayedFrom: string | null; } export interface Summary { @@ -338,7 +337,7 @@ export async function writeReport( .filter((s) => meta.storageState[s]) .map((s) => ` · ${s} storage state ${esc(meta.storageState[s]!)}`) .join("")} - ${meta.rightCache?.replayedFrom ? `· right replayed from a recording of ${esc(meta.rightCache.replayedFrom)}` : ""} + ${meta.rightReplayedFrom ? `· right replayed from a recording of ${esc(meta.rightReplayedFrom)}` : ""}

${section(compared)} diff --git a/src/run.ts b/src/run.ts index 49ccf9c..de6dc78 100644 --- a/src/run.ts +++ b/src/run.ts @@ -208,7 +208,8 @@ export async function run({ } if (!prev && fs.existsSync(cfg.outDir)) { for (const f of fs.readdirSync(cfg.outDir)) { - if (f !== CACHE_DIR) fs.rmSync(path.join(cfg.outDir, f), { recursive: true, force: true }); + if (f !== CACHE_DIR || !cacheRight) + fs.rmSync(path.join(cfg.outDir, f), { recursive: true, force: true }); } } fs.mkdirSync(cfg.outDir, { recursive: true }); @@ -220,7 +221,10 @@ export async function run({ for (const route of routes) { for (const width of cfg.widths) { const had = before.get(`${name(route)}@${width}`); - if (had && reusable(had)) results.push(had); + // A shot measured against a replay stands only for a run that would replay it too. + const replay = had?.status === "compared" && had.rightReplayedFrom; + const fresh = !replay || (cacheRight && Date.now() - Date.parse(replay) < REPLAY_TTL); + if (had && reusable(had) && fresh) results.push(had); else jobs.push({ route, width }); } } @@ -328,7 +332,7 @@ export async function run({ ...made.map((c, i) => write(cropped[i]!.path, c.png)), ]); - recorded.push(shotId(route, width)); + if (!rightShot.failed.length) recorded.push(shotId(route, width)); results.push({ ...entry, status: "compared", @@ -338,6 +342,7 @@ export async function run({ crops: cropped, accepted: false, failed: { left: leftShot.failed, right: rightShot.failed }, + ...(replayedAt ? { rightReplayedFrom: new Date(replayedAt).toISOString() } : {}), }); const { regions, bands, alignedScore, text } = metrics; const dom = elementsOf(metrics); @@ -390,10 +395,22 @@ export async function run({ await launched.browser.close(); } const browserMeta = launched ? browserOf(launched) : prev!.meta.browser; + let unsaved = ""; if (har && !replayedAt) { - const parts = Array.from({ length: cfg.concurrency }, (_, i) => part(i)); - saveRecording(har, parts, recorded, now); + try { + saveRecording( + har, + Array.from({ length: cfg.concurrency }, (_, i) => part(i)), + recorded, + now, + ); + } catch (e) { + unsaved = e instanceof Error ? e.message : String(e); + } } + const replays = results.flatMap((r) => + r.status === "compared" && r.rightReplayedFrom ? [r.rightReplayedFrom] : [], + ); // A rerun overwrites a shot's files in place; drop the ones it no longer writes, ours only. const kept = new Set(results.flatMap(files)); @@ -437,9 +454,7 @@ export async function run({ ...rest, browser: browserMeta, config, - rightCache: har - ? { replayedFrom: replayedAt ? new Date(replayedAt).toISOString() : null } - : null, + rightReplayedFrom: replays.sort()[0] ?? null, }; const reportPath = await writeReport(results, meta, cfg, stale); const resultsPath = writeResults(results, meta, cfg.outDir, stale); @@ -451,10 +466,12 @@ export async function run({ `${results.length - jobs.length} reused, ${names.length - rerunRoutes} route(s) untouched`, ); } - if (har) { + if (meta.rightReplayedFrom) { + console.log(`right: replayed from a recording of ${meta.rightReplayedFrom} (--cache-right)`); + } else if (har) { console.log( - replayedAt - ? `right: replayed from a recording of ${new Date(replayedAt).toISOString()} (--cache-right)` + unsaved + ? `right: loaded live; recording not saved: ${unsaved}` : `right: loaded live and recorded for --cache-right (replayed for ${REPLAY_TTL / 60_000} min)`, ); } diff --git a/tests/unit/cache.test.ts b/tests/unit/cache.test.ts index 73a3f6c..1146bd5 100644 --- a/tests/unit/cache.test.ts +++ b/tests/unit/cache.test.ts @@ -13,17 +13,25 @@ import { cleanupTempDirs, tempDir } from "../helpers/cli.ts"; after(cleanupTempDirs); const browser = { name: "chromium", version: "1" }; -const cfg = (over: Parameters[1] = {}) => - resolveConfig({}, { left: "http://l.test", right: "http://r.test", outDir: "/tmp/o", ...over }); +const cfg = (over: Parameters[1] = {}, file = {}) => + resolveConfig(file, { left: "http://l.test", right: "http://r.test", outDir: "/tmp/o", ...over }); +const entry = (url: string, status = 200) => ({ + request: { method: "GET", url }, + response: { status }, +}); const [route] = parseRoutes(["/a"]); -function record(shots: string[], recordedAt: number) { - const dir = tempDir(); - const parts = [0, 1].map((i) => path.join(dir, `${i}.part`)); +function record( + shots: string[], + recordedAt: number, + entries = [[entry("/a")], [entry("/b")]], + dir = tempDir(), +) { + const file = path.join(dir, "right-x.har"); + const parts = entries.map((_, i) => `${file}.${i}.part`); parts.forEach((p, i) => - fs.writeFileSync(p, JSON.stringify({ log: { version: "1.2", entries: [{ i }] } })), + fs.writeFileSync(p, JSON.stringify({ log: { version: "1.2", entries: entries[i] } })), ); - const file = path.join(dir, "right.har"); saveRecording(file, [...parts, path.join(dir, "missing.part")], shots, recordedAt); return { file, parts }; } @@ -36,10 +44,29 @@ describe("replayable", () => { test("a fresh recording covering every shot replays; the parts merge and go", () => { const { file, parts } = record([a, b], now - 1000); assert.equal(replayable(file, [a], now), now - 1000); - assert.deepEqual(JSON.parse(fs.readFileSync(file, "utf8")).log.entries, [{ i: 0 }, { i: 1 }]); + assert.deepEqual(JSON.parse(fs.readFileSync(file, "utf8")).log.entries, [ + entry("/a"), + entry("/b"), + ]); assert.ok(parts.every((p) => !fs.existsSync(p))); }); + test("errors and repeats are dropped, so a one-off 500 is never replayed", () => { + const { file } = record([a], now, [ + [entry("/a", 500), entry("/a.css", 404)], + [entry("/a"), entry("/a")], + ]); + assert.deepEqual(JSON.parse(fs.readFileSync(file, "utf8")).log.entries, [entry("/a")]); + }); + + test("a new recording replaces every other one beside it", () => { + const dir = tempDir(); + fs.writeFileSync(path.join(dir, "right-old.har"), "{}"); + fs.writeFileSync(path.join(dir, "right-old.har.json"), "{}"); + record([a], now, undefined, dir); + assert.deepEqual(fs.readdirSync(dir).sort(), ["right-x.har", "right-x.har.json"]); + }); + test("a shot it lacks, an expired or future recording, or none at all loads live", () => { assert.equal(replayable(record([a], now).file, [a, b], now), null); assert.equal(replayable(record([a], now - REPLAY_TTL).file, [a], now), null); @@ -61,7 +88,7 @@ describe("keys", () => { assert.equal(shotId(route!, 1440), shotId(paired, 1440), "the left path doesn't matter"); }); - test("the right origin, its storage state and the browser change the file", () => { + test("the right origin, its storage state, blocked URLs and the browser change the file", () => { const dir = tempDir(); const state = path.join(dir, "auth.json"); fs.writeFileSync(state, JSON.stringify({ cookies: [], origins: [] })); @@ -73,6 +100,7 @@ describe("keys", () => { cacheFile(cfg(), { ...browser, version: "2" }), signed, cacheFile(cfg({ storageState: { right: state } }), browser), + cacheFile(cfg({}, { blockUrlPatterns: ["**/ads/**"] }), browser), ]; assert.equal(new Set(files).size, files.length); assert.equal(cacheFile(cfg({ left: "http://x.test" }), browser), files[0]); diff --git a/tests/visual/accepted.test.ts b/tests/visual/accepted.test.ts index 7618b94..1149119 100644 --- a/tests/visual/accepted.test.ts +++ b/tests/visual/accepted.test.ts @@ -10,6 +10,7 @@ */ import assert from "node:assert/strict"; import fs from "node:fs"; +import http from "node:http"; import path from "node:path"; import { after, before, describe, test } from "node:test"; import { cleanupTempDirs, context, renderdiffArgs, runCli, tempDir } from "../helpers/cli.ts"; @@ -254,7 +255,7 @@ describe("--cache-right", () => { assert.equal(first.code, 0, context(first)); assert.match(first.stdout, /^right: loaded live and recorded for --cache-right/m); const was = results(first.outDir); - assert.equal(was.meta.rightCache?.replayedFrom, null); + assert.equal(was.meta.rightReplayedFrom, null); const hits = site.hits("/nav"); const again = await runCli(args(routes, ["--cache-right"]), { outDir: first.outDir }); @@ -262,9 +263,11 @@ describe("--cache-right", () => { assert.equal(site.hits("/nav"), hits + 1, "only the left side is loaded"); assert.match(again.stdout, /^right: replayed from a recording of \S+ \(--cache-right\)$/m); const now = results(again.outDir); - assert.ok(now.meta.rightCache?.replayedFrom, context(again)); + assert.ok(now.meta.rightReplayedFrom, context(again)); for (const route of routes) { - assert.deepEqual(result(now, route, 1440), result(was, route, 1440)); + const { rightReplayedFrom, ...replayed } = result(now, route, 1440); + assert.equal(rightReplayedFrom, now.meta.rightReplayedFrom); + assert.deepEqual(replayed, result(was, route, 1440)); } assert.ok( result(now, "/nav", 1440).bands?.some((b) => b.elements?.[0]?.side === "right"), @@ -278,4 +281,76 @@ describe("--cache-right", () => { assert.equal(site.hits("/nav"), hits + 3, "a shot the recording lacks loads it all live"); assert.match(wider.stdout, /^right: loaded live and recorded/m); }); + + test("a shot measured against a replay is reused only by a run that would replay it", async () => { + const first = await runCli(args(["/"], ["--cache-right"])); + assert.equal(first.code, 0, context(first)); + const again = (extra: string[]) => runCli(args(["/"], extra), { outDir: first.outDir }); + const replayed = await again(["--cache-right"]); + assert.deepEqual(parseSummary(replayed.stdout).clean, ["/"], context(replayed)); + + const kept = await again(["--rerun-failed", "--cache-right"]); + assert.equal(parseSummary(kept.stdout).rerun?.reused, 1, context(kept)); + assert.equal( + results(kept.outDir).meta.rightReplayedFrom, + results(replayed.outDir).meta.rightReplayedFrom, + ); + assert.match(kept.stdout, /^right: replayed from a recording of/m); + + const hits = site.hits("/"); + const live = await again(["--rerun-failed"]); + assert.equal(site.hits("/"), hits + 2, context(live)); + assert.equal(parseSummary(live.stdout).rerun?.shots, 1, context(live)); + assert.equal(results(live.outDir).meta.rightReplayedFrom, null); + assert.doesNotMatch(live.stdout, /^right:/m); + }); + + test("a right side that failed once is not replayed as an error", async () => { + let failed = false; + const target = new URL(site.right); + const proxy = http.createServer((req, res) => { + if (req.url === "/nav" && req.method === "GET" && !failed) { + failed = true; + res.writeHead(500).end("flaky"); + return; + } + const up = http.request( + { + host: target.hostname, + port: target.port, + path: req.url, + method: req.method, + headers: req.headers, + }, + (r) => { + res.writeHead(r.statusCode!, r.headers); + r.pipe(res); + }, + ); + req.pipe(up); + }); + await new Promise((r) => proxy.listen(0, "127.0.0.1", r)); + const right = `http://127.0.0.1:${(proxy.address() as { port: number }).port}`; + try { + // One worker, so /nav@1440 takes the 500 and /nav@375 records a good load beside it. + const flaky = [ + ...args(["/nav"], ["--cache-right"], [1440, 375]), + "--concurrency", + "1", + "--right", + right, + ]; + const first = await runCli(flaky); + assert.equal(first.code, 3, context(first)); + assert.match(first.stdout, /ERROR HTTP 500/, context(first)); + + const replay = [...args(["/nav"], ["--cache-right"], [375]), "--right", right]; + const again = await runCli(replay, { outDir: first.outDir }); + assert.equal(again.code, 0, context(again)); + assert.match(again.stdout, /^right: replayed from/m, context(again)); + } finally { + proxy.closeAllConnections(); + proxy.close(); + } + }); });