From 3ed8d3b7e8c2ef7629d73b619abd93a2c6262500 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Wed, 21 Oct 2026 13:39:23 -0500 Subject: [PATCH 1/2] cvcGL: say whether ImGui's UI changed when a golden fails to settle cvcgl_ari_golden requires the frame after the capture to be pixel-identical ("the UI has settled"). On macOS CI, grid fails that on the first ctest attempt on about half the lanes: 8 px in one column (x=117, y 230..237, the anti-aliased edge of the "d" in the "Inner borders" title) off by 1. The retry passes, and Linux has never shown it (96 runs, 32 in parallel, identical frames). The settle line now also says whether the two frames handed GL the same UI: a fingerprint of ImGui's draw lists (vertices, indices, commands, clip rects, texture ids) and of every texture's size and pixels. A settle failure with the same UI comes from the rasterizer, not the document. It also names the first differing pixel and its two colours. --- src/cvcGL/test/cvcgl_ari_golden.cpp | 94 ++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/src/cvcGL/test/cvcgl_ari_golden.cpp b/src/cvcGL/test/cvcgl_ari_golden.cpp index 8947aaeca..f979ae8e3 100644 --- a/src/cvcGL/test/cvcgl_ari_golden.cpp +++ b/src/cvcGL/test/cvcgl_ari_golden.cpp @@ -28,6 +28,7 @@ #include "streaming_test_util.h" #include +#include #include #include #include @@ -136,7 +137,74 @@ double drawnFraction(const Image &img) { return double(n) / double(img.w * img.h); } +// The first pixel two frames disagree on, with both colours ("" when they are identical). +std::string firstDifference(const Image &a, const Image &b) { + if (a.empty() || b.empty() || a.w != b.w || a.h != b.h) + return ""; + for (int y = 0; y < a.h; ++y) + for (int x = 0; x < a.w; ++x) { + const unsigned char *p = a.at(x, y), *q = b.at(x, y); + if (p[0] != q[0] || p[1] != q[1] || p[2] != q[2]) { + char buf[96]; + std::snprintf(buf, sizeof buf, "first at (%d,%d): %d,%d,%d then %d,%d,%d", x, y, p[0], p[1], + p[2], q[0], q[1], q[2]); + return buf; + } + } + return ""; +} + #ifdef CVC_ENABLE_IMGUI +// What ImGui handed the GL backend for the last rendered frame: its draw lists (vertices, indices, +// commands and their clip rects and textures) and every texture's size and pixels. Two frames with +// equal fingerprints asked GL to draw the same UI, so a pixel difference between them comes from +// the rasterizer, not from the UI. +struct DrawFingerprint { + uint64_t lists = 0, textures = 0; + int vertices = 0; + std::string textureSizes; + bool sameUi(const DrawFingerprint &o) const { + return lists == o.lists && textures == o.textures && vertices == o.vertices; + } +}; + +DrawFingerprint drawFingerprint(cvc::gl::ImGuiOverlay &ui) { + DrawFingerprint fp; + ImGui::SetCurrentContext(ui.imguiContext()); + const ImDrawData *dd = ImGui::GetDrawData(); + if (!dd || !dd->Valid) + return fp; + auto fnv = [](uint64_t &h, const void *data, size_t n) { + const auto *b = static_cast(data); + for (size_t i = 0; i < n; ++i) + h = (h ^ b[i]) * 1099511628211ull; + }; + fp.lists = fp.textures = 1469598103934665603ull; + for (const ImDrawList *dl : dd->CmdLists) { + fnv(fp.lists, dl->VtxBuffer.Data, dl->VtxBuffer.Size * sizeof(ImDrawVert)); + fnv(fp.lists, dl->IdxBuffer.Data, dl->IdxBuffer.Size * sizeof(ImDrawIdx)); + for (const ImDrawCmd &c : dl->CmdBuffer) { + const ImTextureID tex = c.GetTexID(); + fnv(fp.lists, &c.ClipRect, sizeof c.ClipRect); + fnv(fp.lists, &tex, sizeof tex); + fnv(fp.lists, &c.VtxOffset, sizeof c.VtxOffset); + fnv(fp.lists, &c.IdxOffset, sizeof c.IdxOffset); + fnv(fp.lists, &c.ElemCount, sizeof c.ElemCount); + } + fp.vertices += dl->VtxBuffer.Size; + } + if (dd->Textures) + for (const ImTextureData *t : *dd->Textures) { + fnv(fp.textures, &t->Width, sizeof t->Width); + fnv(fp.textures, &t->Height, sizeof t->Height); + if (t->Pixels) + fnv(fp.textures, t->Pixels, static_cast(t->GetSizeInBytes())); + fp.textureSizes += (fp.textureSizes.empty() ? "" : " ") + std::to_string(t->Width) + "x" + + std::to_string(t->Height); + } + return fp; +} + void injectPointer(cvc::gl::ImGuiOverlay &ui, const GoldenCase *gc, int frame) { if (!gc) return; @@ -202,9 +270,15 @@ bool runDocument(const fs::path &doc, const fs::path &goldenDir, const fs::path rt.frame(kDt); const Image actual = cvcgl_golden::fromFrameRGB(view.frameRGB(), view.frameWidth(), view.frameHeight()); +#ifdef CVC_ENABLE_IMGUI + const DrawFingerprint actualUi = drawFingerprint(ui); +#endif rt.frame(kDt); const Image next = cvcgl_golden::fromFrameRGB(view.frameRGB(), view.frameWidth(), view.frameHeight()); +#ifdef CVC_ENABLE_IMGUI + const DrawFingerprint nextUi = drawFingerprint(ui); +#endif for (const std::string &w : rt.take_warnings()) std::printf(" runtime: %s\n", w.c_str()); @@ -219,8 +293,26 @@ bool runDocument(const fs::path &doc, const fs::path &goldenDir, const fs::path check(drawn > 0.01, name + ": the frame is more than the background", std::to_string(100.0 * drawn) + "% of pixels drawn"); const cvcgl_golden::Comparison settled = cvcgl_golden::compare(next, actual, {0, 0.0}); + std::string settledDetail = cvcgl_golden::describe(settled, {0, 0.0}); + const std::string firstDiff = firstDifference(actual, next); + if (!firstDiff.empty()) + settledDetail += "; " + firstDiff; +#ifdef CVC_ENABLE_IMGUI + if (actualUi.sameUi(nextUi)) { + settledDetail += "; ImGui drew the same UI both times (" + std::to_string(nextUi.vertices) + + " vertices, textures " + nextUi.textureSizes + ")"; + } else { + std::string what = actualUi.lists != nextUi.lists ? "draw lists" : ""; + if (actualUi.textures != nextUi.textures) + what += what.empty() ? "textures" : " and textures"; + settledDetail += "; ImGui's " + what + " changed between them (" + + std::to_string(actualUi.vertices) + " -> " + std::to_string(nextUi.vertices) + + " vertices, textures " + actualUi.textureSizes + " -> " + nextUi.textureSizes + + ")"; + } +#endif check(settled.pass, name + ": one more frame draws the same pixels (the UI has settled)", - cvcgl_golden::describe(settled, {0, 0.0})); + settledDetail); // Which golden, and how strictly. const std::string variant = goldenVariant(); From 18a07343aebe31696f731f2e5f15feffcdd0da66 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Wed, 21 Oct 2026 21:14:11 -0500 Subject: [PATCH 2/2] cvcGL: let the golden settle check wait out Apple's rasterizer #761's diagnostic caught the macOS flake on CI run 39654691408 (package-macos / libcvc-debug, attempt 1): grid's two frames differed by 1 in 8 px (first at (39,61): 38,38,38 then 38,39,39) while ImGui had handed GL byte-identical draw lists and textures both times. So the Apple Software Renderer drew the same UI two ways. It has only ever hit the first document in a process, and the later documents in that same process settle at +-0, so it is a transient of the renderer's first frames that this test cannot pin. The settle check keeps +-0. When the next frame differs but the UI fingerprint is identical, the later frame becomes the capture and the test renders another, up to 10 times, and the log says how many it took. A UI that changed still fails at once, and so does a rasterizer that never settles. docs/ARIADNE.md section 12 describes both. --- docs/ARIADNE.md | 12 ++++++ src/cvcGL/test/cvcgl_ari_golden.cpp | 66 ++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/docs/ARIADNE.md b/docs/ARIADNE.md index 7407639ce..3efb1973c 100644 --- a/docs/ARIADNE.md +++ b/docs/ARIADNE.md @@ -1236,6 +1236,18 @@ A capture is made repeatable by: The test then renders one more frame and requires it to be identical, so a UI that is still changing fails as unsettled rather than as a flaky mismatch. +To tell the UI from the rasterizer, each of the two frames is fingerprinted with what ImGui +handed GL: its draw lists (vertices, indices, commands, clip rects, texture ids) and every +texture's size and pixels. The settle line says whether the UI was the same both times, and +names the first differing pixel. + +macOS CI's Apple Software Renderer sometimes draws the same UI two ways in a process's first +frames. It has only ever hit the first document. In 2026-10 it moved 8 pixels by 1 between +frames whose fingerprints were identical. So when the UI is identical and only pixels differ, +the later frame becomes the capture and the test renders another, up to 10 times, until two +frames agree exactly. The log says how many extra frames that took. A UI that changed fails at +once, as does a rasterizer that never settles. + **Comparing.** A pixel mismatches when a channel differs by more than 40. A document passes while at most 0.002% of its pixels (13) mismatch. The goldens are made under Mesa llvmpipe, the rasterizer of the Linux CI job (Xvfb + cvcpkg's `mesa`). These limits were measured: diff --git a/src/cvcGL/test/cvcgl_ari_golden.cpp b/src/cvcGL/test/cvcgl_ari_golden.cpp index f979ae8e3..17a35ca0b 100644 --- a/src/cvcGL/test/cvcgl_ari_golden.cpp +++ b/src/cvcGL/test/cvcgl_ari_golden.cpp @@ -10,6 +10,9 @@ // least 700 px on its short side, so the small-screen rule never closes them), ImGui's frame delta // pinned, and several frames at a fixed dt before the capture, so new windows have finished // appearing. The test then renders one frame more and requires it to be identical to the capture. +// When ImGui drew exactly the same UI in both and only the rasterizer moved (the Apple Software +// Renderer, in a process's first frames), it renders up to kRasterSettleFrames more until two +// agree. // // Goldens are made under Mesa llvmpipe, the rasterizer of the Linux and Windows CI jobs. Other // rasterizers, and other Mesa versions, draw anti-aliased edges and shading a little differently, @@ -56,6 +59,7 @@ namespace { constexpr int kWidth = 960, kHeight = 720; // short side >= 700: the overlay keeps its panels open constexpr double kDt = 1.0 / 30.0; // the host frame step AND ImGui's frame delta constexpr int kMinFrames = 4; // frames before the capture (ImGui auto-fit needs 2) +constexpr int kRasterSettleFrames = 10; // more frames for a rasterizer drawing one UI two ways const double kBackground[3] = {0.09, 0.10, 0.12}; // One tolerance for every rasterizer, from these measurements against the goldens (Mesa 25.3.6 @@ -154,22 +158,25 @@ std::string firstDifference(const Image &a, const Image &b) { return ""; } -#ifdef CVC_ENABLE_IMGUI // What ImGui handed the GL backend for the last rendered frame: its draw lists (vertices, indices, // commands and their clip rects and textures) and every texture's size and pixels. Two frames with // equal fingerprints asked GL to draw the same UI, so a pixel difference between them comes from // the rasterizer, not from the UI. struct DrawFingerprint { + bool valid = false; // false: no draw data (or no ImGui in this build), so nothing is known uint64_t lists = 0, textures = 0; int vertices = 0; std::string textureSizes; bool sameUi(const DrawFingerprint &o) const { - return lists == o.lists && textures == o.textures && vertices == o.vertices; + return valid && o.valid && lists == o.lists && textures == o.textures && vertices == o.vertices; } }; DrawFingerprint drawFingerprint(cvc::gl::ImGuiOverlay &ui) { DrawFingerprint fp; +#ifndef CVC_ENABLE_IMGUI + (void)ui; +#else ImGui::SetCurrentContext(ui.imguiContext()); const ImDrawData *dd = ImGui::GetDrawData(); if (!dd || !dd->Valid) @@ -202,9 +209,12 @@ DrawFingerprint drawFingerprint(cvc::gl::ImGuiOverlay &ui) { fp.textureSizes += (fp.textureSizes.empty() ? "" : " ") + std::to_string(t->Width) + "x" + std::to_string(t->Height); } + fp.valid = true; +#endif return fp; } +#ifdef CVC_ENABLE_IMGUI void injectPointer(cvc::gl::ImGuiOverlay &ui, const GoldenCase *gc, int frame) { if (!gc) return; @@ -267,24 +277,41 @@ bool runDocument(const fs::path &doc, const fs::path &goldenDir, const fs::path // no X display, which imgui's GLX-based loader cannot reach) nothing here can be tested. if (!ui.enabled()) return false; - rt.frame(kDt); - const Image actual = - cvcgl_golden::fromFrameRGB(view.frameRGB(), view.frameWidth(), view.frameHeight()); -#ifdef CVC_ENABLE_IMGUI - const DrawFingerprint actualUi = drawFingerprint(ui); -#endif - rt.frame(kDt); - const Image next = - cvcgl_golden::fromFrameRGB(view.frameRGB(), view.frameWidth(), view.frameHeight()); -#ifdef CVC_ENABLE_IMGUI - const DrawFingerprint nextUi = drawFingerprint(ui); -#endif + // The capture, then one frame more, which must draw the same pixels. A UI that is still changing + // fails here. The one other way two frames differ is the rasterizer: ImGui handed GL exactly the + // same UI (the same fingerprint) and some pixels still moved. macOS CI's Apple Software Renderer + // does that, by 1 in a few pixels, in a process's first frames. Then the later frame becomes the + // capture and one more is drawn, up to kRasterSettleFrames times, and the log says so. + auto render = [&] { + rt.frame(kDt); + return cvcgl_golden::fromFrameRGB(view.frameRGB(), view.frameWidth(), view.frameHeight()); + }; + Image actual = render(); + DrawFingerprint actualUi = drawFingerprint(ui); + Image next; + DrawFingerprint nextUi; + cvcgl_golden::Comparison settled; + std::string rasterNoise; // the first same-UI difference, when the rasterizer needed more frames + int rasterFrames = 0; + for (;;) { + next = render(); + nextUi = drawFingerprint(ui); + settled = cvcgl_golden::compare(next, actual, {0, 0.0}); + if (settled.pass || !actualUi.sameUi(nextUi) || rasterFrames == kRasterSettleFrames) + break; + if (rasterNoise.empty()) + rasterNoise = + cvcgl_golden::describe(settled, {0, 0.0}) + "; " + firstDifference(actual, next); + ++rasterFrames; + actual = std::move(next); + actualUi = nextUi; + } for (const std::string &w : rt.take_warnings()) std::printf(" runtime: %s\n", w.c_str()); const std::string renderer = glRenderer(view); const bool llvmpipe = renderer.find("llvmpipe") != std::string::npos; - std::printf(" %d frames at %dx%d on %s\n", frames + 2, kWidth, kHeight, + std::printf(" %d frames at %dx%d on %s\n", frames + 2 + rasterFrames, kWidth, kHeight, renderer.empty() ? "(unknown renderer)" : renderer.c_str()); check(!actual.empty() && actual.w == kWidth && actual.h == kHeight, @@ -292,16 +319,14 @@ bool runDocument(const fs::path &doc, const fs::path &goldenDir, const fs::path const double drawn = drawnFraction(actual); check(drawn > 0.01, name + ": the frame is more than the background", std::to_string(100.0 * drawn) + "% of pixels drawn"); - const cvcgl_golden::Comparison settled = cvcgl_golden::compare(next, actual, {0, 0.0}); std::string settledDetail = cvcgl_golden::describe(settled, {0, 0.0}); const std::string firstDiff = firstDifference(actual, next); if (!firstDiff.empty()) settledDetail += "; " + firstDiff; -#ifdef CVC_ENABLE_IMGUI if (actualUi.sameUi(nextUi)) { settledDetail += "; ImGui drew the same UI both times (" + std::to_string(nextUi.vertices) + " vertices, textures " + nextUi.textureSizes + ")"; - } else { + } else if (actualUi.valid && nextUi.valid) { std::string what = actualUi.lists != nextUi.lists ? "draw lists" : ""; if (actualUi.textures != nextUi.textures) what += what.empty() ? "textures" : " and textures"; @@ -310,7 +335,10 @@ bool runDocument(const fs::path &doc, const fs::path &goldenDir, const fs::path " vertices, textures " + actualUi.textureSizes + " -> " + nextUi.textureSizes + ")"; } -#endif + if (rasterFrames > 0) + settledDetail += std::string("; with the same UI, the rasterizer ") + + (settled.pass ? "settled after " : "had not settled after ") + + std::to_string(rasterFrames) + " more frame(s) (first: " + rasterNoise + ")"; check(settled.pass, name + ": one more frame draws the same pixels (the UI has settled)", settledDetail);