CI add visual regression tests

cortex/export/headless.py

@@ -99,6 +99,39 @@ def _wait_for_viewer_loaded(handle, timeout: float = 60.0) -> None:
 # --------------------------------------------------------------------------- #
 
 
+#: Browser messages that mean WebGL itself failed, as opposed to any of the
+#: unrelated javascript a page may log. Kept deliberately narrow.
+#:
+#: Checking *all* browser errors is not usable: a healthy viewer already logs a
+#: console.error for the Leap Motion websocket it cannot reach
+#: (ws://127.0.0.1:6437), so anything failing on "any error" fails every run.
+#: That is why the existing assertions in the test suite filter to [pageerror].
+#:
+#: These two are unambiguous failures:
+#:   - "Could not initialise shader" is three.js reporting a shader that compiled
+#:     but failed to *link*. It is a console.error, not an exception, so nothing
+#:     raises and the render silently comes back as a blank canvas. This is
+#:     exactly how Vertex2D broke (gh-714).
+#:   - "Error creating WebGL context" is thrown, so it also arrives as a
+#:     [pageerror]; matching the text covers it either way.
+#:
+#: gl.getProgramInfoLog / gl.getShaderInfoLog warnings are deliberately NOT
+#: listed. Drivers emit those benignly, so matching them would reintroduce the
+#: false positives this list exists to avoid.
+WEBGL_FAILURE_PATTERNS = (
+    "THREE.WebGLProgram: Could not initialise shader",
+    "Error creating WebGL context",
+)
+
+
+def webgl_failures(browser_errors: list[str]) -> list[str]:
+    """Return only those browser messages that indicate WebGL itself failed."""
+    return [
+        e for e in browser_errors
+        if any(pattern in e for pattern in WEBGL_FAILURE_PATTERNS)
+    ]
+
+
 class _PlaywrightThread:
     """Manages the Playwright lifecycle on a private daemon thread.
 

cortex/export/save_views.py

@@ -196,6 +196,25 @@ def save_3d_views(
                 )
             time.sleep(1)
 
+            # A WebGL failure does not raise on its own: three.js reports a
+            # shader that failed to link via console.error, so the png is still
+            # written and simply comes back blank, which is indistinguishable
+            # from a successful render. Fail loudly instead. Only genuine WebGL
+            # failures are matched -- a healthy viewer logs unrelated console
+            # errors (the Leap Motion websocket), so checking every browser
+            # error would fail every run.
+            pw_thread = getattr(handle, "_pw_thread", None)
+            if pw_thread is not None:
+                from cortex.export.headless import webgl_failures
+
+                failures = webgl_failures(pw_thread.browser_errors)
+                if failures:
+                    raise RuntimeError(
+                        f"WebGL failed while rendering {view_name!r}/{surface!r}; "
+                        f"{file_name!r} is likely blank.\n  "
+                        + "\n  ".join(sorted(set(failures)))
+                    )
+
             # Trim transparent edges
             if trim:
                 try:

cortex/tests/reference_images/README.md

@@ -0,0 +1,159 @@
+# Reference images
+
+Stored renders that `cortex/tests/test_visual_regression.py` asserts against, so
+a change in rendering output fails a test instead of needing to be spotted by
+eye.
+
+## `alpha_dataviews/`
+
+Twelve images: each of the six public dataview classes (`Volume`, `Vertex`,
+`Volume2D`, `Vertex2D`, `VolumeRGB`, `VertexRGB`) rendered through both paths --
+`quickflat_*` via `cortex.quickshow` (matplotlib) and `webgl_*` via
+`cortex.export.plot_panels` (headless WebGL).
+
+Between them they exercise every way pycortex encodes alpha: `Volume`/`Vertex` are
+a no-alpha baseline, `Volume2D`/`Vertex2D` use the 2D alpha colormap
+`RdBu_r_alpha`, and `VolumeRGB`/`VertexRGB` use the native `alpha=` keyword. All
+six also composite the curvature underlay.
+
+## `nan_dataviews/`
+
+Twelve images, laid out exactly as `alpha_dataviews/`: the same six dataview
+classes, but with NaNs standing in for "missing data" over roughly half of the
+volume/surface (the primary data channel, not the alpha channel). Both
+renderers are expected to draw those elements as fully transparent, falling
+through to the curvature underlay, rather than mapping NaN through the
+colormap as if it were a real value.
+
+## `nan_alpha_dataviews/`
+
+Four images: `VolumeRGB` and `VertexRGB` -- the only dataviews taking an explicit
+`alpha=` -- with the NaNs in the **alpha map** rather than in the data. That is a
+distinct path, since alpha is not colour-mapped but used directly as a blend
+weight, so the NaN reaches the compositing arithmetic rather than a colormap
+lookup. Both renderers currently draw those elements fully transparent.
+
+This behaviour is not settled: work to unify NaN and alpha handling across
+quickflat, WebGL and the RGB dataviews (`cb976270`, not in this branch's history)
+changes how the surviving RGB is blended without changing the transparency
+itself. Expect these four to need regenerating if that lands.
+
+## Within- vs cross-renderer checks
+
+All three tests -- `..._alpha_dataviews`, `..._nan_dataviews` and
+`..._nan_alpha_dataviews` -- check every render two ways:
+
+- **Within-renderer**: `quickflat_*`/`webgl_*` against their own stored
+  reference above, at a tight tolerance. Four criteria, all of which must pass:
+  `MAX_MEAN_ABS_DIFF`, `MAX_FRACTION_DIFFERING`,
+  `MAX_FRACTION_GROSSLY_DIFFERING` and `MAX_SSIM_LOSS`. They are complementary,
+  not redundant -- the mean is blind to a change that moves a few pixels a long
+  way, the gross fraction is blind to a broad low-amplitude shift, and SSIM is
+  blind to a channel permutation. See the test file for the calibration. This is
+  what catches a regression in one renderer's pipeline.
+- **Cross-renderer**: quickflat vs webgl for the *same* dataview, diffed
+  directly against each other with no stored fixture, after the affine
+  correction below puts them on a common grid, at a looser tolerance
+  (`CROSS_MAX_MEAN_ABS_DIFF` / `CROSS_MAX_FRACTION_DIFFERING`). matplotlib and
+  Three.js still differ in colormap sampling and anti-aliasing, so this can't
+  use the within-renderer tolerance, but it still catches a large disagreement
+  (wrong colormap, dropped alpha, swapped channels) between the two paths even
+  if a stale or wrongly-regenerated reference would otherwise hide it.
+
+## The cross-renderer affine correction
+
+The two renderers do not share a pixel grid: each computes its own trim/extent,
+so webgl's flatmap lands in quickflat's frame off by a fixed anisotropic scale
+(x 0.9690, y 0.9365) plus a ~7 px translation. `_check_cross_renderer` undoes
+that before diffing, via the `CROSS_RENDERER_*` constants in
+`cortex/tests/test_visual_regression.py`. Without it the cross-renderer diff
+sits at mean|diff| ~19, which swamps everything a regression would do; with it,
+~2-3.
+
+Note this is *not* perspective distortion, despite webgl rendering through a
+`THREE.PerspectiveCamera` (FOV 45°, `axes3d.js`). The flatmap view looks
+straight down at a planar surface, where a pinhole projection degenerates to a
+uniform scale — fitting a full homography returns a projective row of
+`[~0, ~0, 1]` and ~0° rotation, i.e. it collapses to the affine. Chasing a
+reverse perspective projection is a dead end.
+
+To re-derive the constants:
+
+```
+uv run --with opencv-python-headless \
+    python cortex/tests/reference_images/fit_cross_renderer_affine.py
+```
+
+It renders curvature-only content (a `VolumeRGB` with `alpha=0` everywhere, so
+only the curvature underlay is drawn — the one layer both paths composite
+identically, which isolates the coordinate-frame mismatch from any colormap
+difference), fits webgl → quickflat with `cv2.findTransformECC`, and prints the
+constants ready to paste. OpenCV is only needed for the fit, so it is not a
+project dependency — hence `--with`.
+
+Run it if the cross-renderer check starts failing across the board with a
+mean|diff| near 19 rather than ~2, which is what losing the correction looks
+like — e.g. after a change to `plot_panels`' figure composition, to quickflat's
+`height=`/`dpi=`, or to either renderer's trim logic. The script's render
+settings mirror the test's; if the test's change, update the script to match or
+the fit will not apply. It also writes `fit_affine_residual.png`, which should
+show a faint sulcal outline only — broad structure there means the fit didn't
+take.
+
+## Provenance
+
+Generated on this branch, on top of `claude/issue-714-irf5g0` (the Vertex2D
+blank-render fix).
+
+They can **no longer be reproduced at `5af26a86`**, which earlier revisions of
+this file named. Regenerating there and testing against the current tree fails
+the six vertex-based cases: `#679` moved every pure-Vertex flatmap's trim box
+from 594 to 596px, and that is intended new geometry which its follow-up fix
+does not revert. The failures are a genuine content change, not a resampling
+artifact -- optimal affine alignment removes only 12% of the difference, and the
+difference is concentrated in the interior rather than at the silhouette.
+
+All twelve *quickflat* references are byte-identical at either commit; only the
+webgl ones moved.
+
+| | |
+| --- | --- |
+| chromium | 151.0.7922.34 (headless shell, SwiftShader software rendering) |
+| matplotlib | 3.11.1 |
+| pillow | 12.3.0 |
+
+## Format
+
+Lossless WebP (`method=6`, `quality=100`, `exact=True`): bit-exact after decode,
+and 59% the size of optimized PNG (669 KiB versus 1129 KiB for the set).
+
+## Distribution
+
+These are test fixtures with no runtime use, so they are **kept out of the wheel**
+(`exclude_package_data` in `setup.py`) and **kept in the source tarball**
+(`MANIFEST.in`'s `recursive-include cortex *`). A build from source can therefore
+run the test; a `pip install` does not carry ~700 KiB of PNGs into site-packages
+for data no user will read.
+
+The test skips, rather than fails, when the images are absent, so a test run
+against an installed wheel degrades gracefully.
+
+## Regenerating
+
+The renders are deterministic: repeated runs on one machine produce bit-identical
+output, including the WebGL ones under software rendering. They are, however,
+coupled to the Chromium and matplotlib builds above, so a browser or matplotlib
+upgrade can shift anti-aliasing and rasterization slightly. The test's tolerances
+absorb that; if an upgrade moves output beyond them, inspect the `diff_*.png`
+files the failure writes, confirm the change is cosmetic, then:
+
+```
+REGENERATE_REFERENCE_IMAGES=1 pytest cortex/tests/test_visual_regression.py
+```
+
+(This regenerates all three directories in one run, since all three tests live in
+that file. It does not touch the affine correction, which is fitted separately —
+see above.)
+
+Review the resulting diff before committing -- regenerating is how a real
+regression gets silently blessed.

cortex/tests/reference_images/fit_cross_renderer_affine.py

@@ -0,0 +1,137 @@
+"""Re-derive the cross-renderer affine transform used by test_visual_regression.py.
+
+quickflat and webgl do not agree on a common pixel grid: each computes its own
+trim/extent, so webgl's flatmap lands in quickflat's frame off by a fixed
+anisotropic scale plus a translation. ``_check_cross_renderer`` corrects for
+that before diffing, using the ``CROSS_RENDERER_*`` constants in
+``cortex/tests/test_visual_regression.py``. This script recomputes them.
+
+Run it if the cross-renderer check starts failing everywhere at once with a
+large mean|diff| (~19 rather than ~2), which is what losing the correction looks
+like -- e.g. after a change to ``plot_panels``' figure composition, to
+quickflat's ``height=``/``dpi=``, or to either renderer's trim logic.
+
+    uv run --with opencv-python-headless \
+        python cortex/tests/reference_images/fit_cross_renderer_affine.py
+
+OpenCV is only needed for the fit, so it is deliberately not a project
+dependency -- hence ``--with``.
+
+Method: render curvature-only content (a VolumeRGB with alpha=0 everywhere, so
+the data layer is fully transparent and only the curvature underlay is drawn)
+through both renderers. Curvature is the one layer both paths composite
+identically, so the fit measures the coordinate-frame mismatch rather than any
+dataview-specific colormap difference. Then fit webgl -> quickflat with
+``cv2.findTransformECC``, first as a full homography to confirm there is no real
+projective component, then as an affine to read off the constants.
+
+Note on conventions: ``cv2.warpAffine(..., WARP_INVERSE_MAP)`` and PIL's
+``Image.AFFINE`` use the same inverse mapping (output pixel -> source coords),
+so the fitted matrix rows drop straight into PIL's coefficient tuple in
+``_check_cross_renderer`` with no inversion or transpose.
+"""
+
+import numpy as np
+
+import cortex
+import cortex.export
+
+SUBJ = "S1"
+XFMNAME = "fullhead"
+
+# Must match what test_visual_regression.py's _render_and_check_dataview does,
+# or the fitted transform will not apply to the renders the test produces.
+QUICKFLAT_HEIGHT = 256
+QUICKFLAT_DPI = 80
+WEBGL_FIGSIZE = (6, 3)
+WEBGL_WINDOWSIZE = (512, 384)
+WEBGL_SLEEP = 10
+
+FLATMAP_PANEL = [
+    {"extent": [0.0, 0.0, 1.0, 1.0], "view": {"angle": "flatmap", "surface": "flatmap"}}
+]
+
+
+def _curvature_only_dataview():
+    """A dataview whose data layer is fully transparent, leaving only curvature."""
+    zeros = np.zeros((31, 100, 100))
+    chan = lambda: cortex.Volume(zeros, SUBJ, XFMNAME, vmin=0, vmax=1)
+    return cortex.VolumeRGB(
+        chan(), chan(), chan(), SUBJ, XFMNAME, alpha=chan(),
+    )
+
+
+def main():
+    import cv2
+    import matplotlib.pyplot as plt
+    from PIL import Image
+
+    view = _curvature_only_dataview()
+
+    qf_path = "fit_affine_quickflat.png"
+    fig = cortex.quickshow(
+        view, with_curvature=True, with_rois=False, with_labels=False,
+        with_colorbar=False, with_sulci=False, with_borders=False,
+        height=QUICKFLAT_HEIGHT,
+    )
+    fig.savefig(qf_path, bbox_inches="tight", pad_inches=0, dpi=QUICKFLAT_DPI)
+    plt.close(fig)
+
+    wg_path = "fit_affine_webgl.png"
+    fig = cortex.export.plot_panels(
+        view, panels=FLATMAP_PANEL, figsize=WEBGL_FIGSIZE,
+        windowsize=WEBGL_WINDOWSIZE, save_name=wg_path, sleep=WEBGL_SLEEP,
+        viewer_params=dict(labels_visible=[], overlays_visible=[]),
+        headless=True,
+    )
+    plt.close(fig)
+
+    qf_im = Image.open(qf_path).convert("L")
+    wg_im = Image.open(wg_path).convert("L")
+    print(f"quickflat {qf_im.size}, webgl {wg_im.size}")
+
+    # Fit in quickflat's frame, which is what _check_cross_renderer diffs in.
+    qf = np.asarray(qf_im).astype(np.float32) / 255.0
+    wg = np.asarray(wg_im.resize(qf_im.size, Image.BILINEAR)).astype(np.float32) / 255.0
+    h, w = qf.shape
+    criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 5000, 1e-6)
+
+    print(f"\nmean|diff| with a plain resize and no correction: "
+          f"{np.abs(qf - wg).mean() * 255:.3f}")
+
+    # Homography first, purely as a check that there is nothing projective to
+    # undo. webgl uses a THREE.PerspectiveCamera, but the flatmap view looks
+    # straight down at a planar surface, where a pinhole projection degenerates
+    # to a uniform scale -- so the bottom row should come out as [0, 0, 1].
+    hom, _ = np.eye(3, 3, dtype=np.float32), None
+    _, hom = cv2.findTransformECC(qf, wg, hom, cv2.MOTION_HOMOGRAPHY, criteria, None, 5)
+    print(f"\nhomography projective row: [{hom[2, 0]:.3e}, {hom[2, 1]:.3e}, {hom[2, 2]:.3f}]")
+    print("  near [0, 0, 1] => no real perspective distortion; affine is sufficient")
+
+    aff = np.eye(2, 3, dtype=np.float32)
+    cc, aff = cv2.findTransformECC(qf, wg, aff, cv2.MOTION_AFFINE, criteria, None, 5)
+    aligned = cv2.warpAffine(
+        wg, aff, (w, h), flags=cv2.INTER_LINEAR + cv2.WARP_INVERSE_MAP
+    )
+    mask = aligned != 0
+    print(f"\naffine fit: correlation={cc:.4f}")
+    print(f"mean|diff| after correction: {np.abs(qf[mask] - aligned[mask]).mean() * 255:.3f}")
+
+    (a, b, tx), (c, d, ty) = aff[0], aff[1]
+    rotation = np.degrees(np.arctan2(c, a))
+    print(f"rotation={rotation:.4f} deg (expect ~0), shear terms b={b:.2e} c={c:.2e}")
+
+    print("\nPaste into cortex/tests/test_visual_regression.py:\n")
+    print(f"CROSS_RENDERER_SCALE_X = {np.hypot(a, c):.4f}")
+    print(f"CROSS_RENDERER_SCALE_Y = {np.hypot(b, d):.4f}")
+    print(f"CROSS_RENDERER_TRANSLATE_X_FRAC = {tx:.4f} / {w}")
+    print(f"CROSS_RENDERER_TRANSLATE_Y_FRAC = {ty:.4f} / {h}")
+
+    amp = np.clip(np.abs(qf - aligned) * 255 * 4, 0, 255).astype("uint8")
+    Image.fromarray(amp).save("fit_affine_residual.png")
+    print("\nresidual (4x amplified) written to fit_affine_residual.png -- expect a")
+    print("faint sulcal outline only; broad structure means the fit did not take")
+
+
+if __name__ == "__main__":
+    main()