From 39014b67efafb93b18efeceae9c22c177c8bd822 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Aug 2026 15:03:15 +0000 Subject: [PATCH 1/5] fix: stamp the small-datasets regime into FITS headers at the writer funnel PYAUTO_SMALL_DATASETS=1 caps simulated datasets to a reduced resolution, but nothing on disk recorded which regime a dataset was written under. A capped dataset could therefore survive into a later full-resolution run and be loaded silently -- the root cause of autolens_workspace_test#260. PyAutoArray#471 closed the imaging half of that by inferring the regime from data.fits's shape. Inference cannot reach interferometer datasets: their visibility count is fixed by the committed uv file while the real-space grid behind it is capped, so a capped run writes identical NAXIS with different values and trips no assertion at all. A silent wrong answer in a likelihood is worse than a loud one. Record the regime at write time instead. A card written by the same call that writes the data is truthful by construction, needs no call-site changes, and does not depend on the data looking different -- the only discriminant that can reach the interferometer case. Stamped at BOTH funnel points, which is load-bearing rather than belt-and-braces: - hdu_list_for_output_from covers the 14 sites in PyAutoGalaxy/PyAutoLens that build an HDUList here and then call hdu_list.writeto directly, plus the 4 that hand it to PyAutoFit's paths.save_fits (verified to writeto the list as-is, so the stamp survives into aggregator output). - write_hdu_list covers HDULists built elsewhere. The public API steers callers this way: hdu_list_for_output_from is re-exported as aa./ag./al.hdu_list_for_output_from while write_hdu_list is re-exported by none of them. Stamping only output_to_fits, as the issue proposed, would have missed 16 of the 18 library write sites. The card is written in BOTH regimes (T capped, F full) rather than only when capped. Absence then means "unknown" -- a file written before this stamp existed -- and readers fall back to their legacy heuristic. Absence must never be read as "full resolution", or every pre-stamp dataset would masquerade as full and the original bug would return. Header assignment rather than append keeps the two stamp points idempotent, so output_to_fits (which passes through both) writes exactly one card. The added card is byte-size neutral in practice: a FITS header block holds 36 cards and a real dataset header carries ~10, so file sizes are unchanged and the byte-size diagnostics used in autolens_workspace_test#260 still hold. --- autonerves/fitsable.py | 57 +++++++++++++++ autonerves/test_mode.py | 20 ++++++ test_autonerves/test_fitsable.py | 119 +++++++++++++++++++++++++++++++ 3 files changed, 196 insertions(+) diff --git a/autonerves/fitsable.py b/autonerves/fitsable.py index 16876eb..4bed148 100644 --- a/autonerves/fitsable.py +++ b/autonerves/fitsable.py @@ -14,6 +14,58 @@ from typing import Dict, Optional, Union, List +SMALL_DATASETS_HEADER_KEY = "SMALLDAT" +SMALL_DATASETS_HEADER_COMMENT = "PYAUTO_SMALL_DATASETS active at write time" + + +def stamp_small_datasets_regime(header): + """ + Record the small-datasets regime in a FITS ``header``, in place. + + ``PYAUTO_SMALL_DATASETS=1`` caps simulated datasets to a reduced + resolution. Nothing else on disk records that fact, so a dataset written + by a capped run can survive into a later full-resolution run and be loaded + silently -- the root cause of autolens_workspace_test#260. + + Writing the regime *here*, in the same call that writes the data, makes the + stamp **truthful by construction**: it cannot disagree with the file it + sits in, and there is no stamped-but-empty-directory failure mode. That is + the property a marker file written before simulation cannot have. + + Unlike a shape heuristic it also does not depend on the data looking + different, which is what makes it the only discriminant able to catch + capped **interferometer** datasets: their visibility count is fixed by the + committed uv file while the real-space grid behind it is capped, so a + capped run writes identical ``NAXIS`` with different values and trips no + assertion at all. + + The card is written on **every** FITS the stack writes, in both regimes: + + - ``SMALLDAT = T`` -- written under ``PYAUTO_SMALL_DATASETS=1`` (capped). + - ``SMALLDAT = F`` -- written at full resolution. + - **absent** -- unknown. Written before this stamp existed, or by another + tool. Readers must fall back to their legacy heuristic and, above all, + must never read absence as "full resolution". + + Recording ``F`` explicitly rather than relying on absence is the whole + point of the always-write: it lets a reader distinguish "known full" from + "no idea", and only the first of those is safe to act on. See + ``autoarray.util.dataset_util.should_simulate``, whose predicate ends in + ``shutil.rmtree``. + + Assignment (not ``append``) is deliberate -- it is idempotent, so the two + call sites below can both stamp the same header without duplicating the + card. + """ + from autonerves.test_mode import small_datasets + + header[SMALL_DATASETS_HEADER_KEY] = ( + small_datasets(), + SMALL_DATASETS_HEADER_COMMENT, + ) + return header + + def hdu_list_for_output_from( values_list: List[np.ndarray], header_dict: Optional[dict] = None, @@ -68,6 +120,8 @@ def hdu_list_for_output_from( except ValueError: header.append((key_str, float(value), [""])) + stamp_small_datasets_regime(header) + for i, values in enumerate(values_list): if ext_name_list is not None: @@ -136,6 +190,9 @@ def write_hdu_list(hdu_list, file_path, overwrite=False): overwrite : bool If ``True`` an existing file is replaced. """ + if len(hdu_list) > 0: + stamp_small_datasets_regime(hdu_list[0].header) + file_path = Path(file_path) file_path.parent.mkdir(parents=True, exist_ok=True) if overwrite and file_path.is_file(): diff --git a/autonerves/test_mode.py b/autonerves/test_mode.py index 030a011..e0474b6 100644 --- a/autonerves/test_mode.py +++ b/autonerves/test_mode.py @@ -214,3 +214,23 @@ def with_test_mode_segment(base: Path) -> Path: PosixPath('output/test_mode/results_folder') """ return base / "test_mode" if is_test_mode() else base + + +def small_datasets(): + """ + Return True if the small-datasets regime is active. + + ``PYAUTO_SMALL_DATASETS=1`` caps simulated datasets to a reduced + resolution (``SMALL_DATASETS_SHAPE_NATIVE = (16, 16)`` in + ``autoarray.util.dataset_util``) so smoke runs stay fast. The cap changes + what a simulator writes to disk, which makes it a *provenance* fact about + every file written while it is active -- see + :func:`autonerves.fitsable.stamp_small_datasets_regime`, which records it + in the FITS header of every array the stack writes. + + The env var is deliberately compared against the exact string ``"1"``, + matching every other ``PYAUTO_*`` switch in this module and the readers in + ``autoarray.util.dataset_util``. ``"0"``, ``"true"`` and the unset case all + mean "full resolution". + """ + return os.environ.get("PYAUTO_SMALL_DATASETS") == "1" diff --git a/test_autonerves/test_fitsable.py b/test_autonerves/test_fitsable.py index e243d53..424f86f 100644 --- a/test_autonerves/test_fitsable.py +++ b/test_autonerves/test_fitsable.py @@ -93,3 +93,122 @@ def test__header_obj_from(): assert isinstance(header_obj, fits.header.Header) assert header_obj["BITPIX"] == -64 + + +""" +__small-datasets regime stamp (PyAutoNerves#153)__ + +`PYAUTO_SMALL_DATASETS=1` caps simulated datasets to a reduced resolution. +Nothing else on disk records that, so a capped dataset can survive into a later +full-resolution run and be loaded silently (autolens_workspace_test#260). The +stamp records the regime in the same call that writes the data, which makes it +truthful by construction and -- unlike a shape heuristic -- able to catch +corruption that leaves the shape unchanged. + +Both funnels are covered below because they are genuinely separate paths: +`output_to_fits` builds and writes in one call, while the multi-HDU dataset +writers in PyAutoArray build via `hdu_list_for_output_from` and write via +`write_hdu_list`, never touching `output_to_fits`. +""" + +KEY = fitsable.SMALL_DATASETS_HEADER_KEY + + +def _stamp(file_path, hdu=0): + with fits.open(file_path) as hdu_list: + return hdu_list[hdu].header.get(KEY) + + +def test__output_to_fits__stamps_the_regime(tmp_path, monkeypatch): + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "small.fits") + assert _stamp(tmp_path / "small.fits") is True + + monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False) + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "full.fits") + assert _stamp(tmp_path / "full.fits") is False + + +def test__stamp_is_written_in_both_regimes__absence_is_never_full( + tmp_path, monkeypatch +): + # The full regime is recorded EXPLICITLY as F rather than by omission. + # That is the whole point: a reader can then tell "known full" from "no + # idea", and only the first is safe to act on. If F were encoded as + # absence, every legacy dataset would masquerade as full resolution and + # the original bug would come straight back. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "0") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "zero.fits") + + assert KEY in fits.open(tmp_path / "zero.fits")[0].header + assert _stamp(tmp_path / "zero.fits") is False + + +def test__stamp_round_trips_as_a_fits_boolean_not_a_string(tmp_path, monkeypatch): + # Readers distinguish True/False/absent and must never coerce: bool("F") + # is True. Pin the on-disk type so a future change to the header-writing + # path cannot silently downgrade the card to a string or a float. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "t.fits") + + value = fits.open(tmp_path / "t.fits")[0].header[KEY] + assert isinstance(value, bool) + assert "SMALLDAT= T" in str( + fits.open(tmp_path / "t.fits")[0].header.cards[KEY] + ) + + +def test__multi_hdu_funnel__stamps_every_hdu(tmp_path, monkeypatch): + # The path PyAutoArray's fits_imaging/fits_interferometer take when given a + # single `file_path` -- it bypasses output_to_fits entirely. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + hdu_list = fitsable.hdu_list_for_output_from( + values_list=[np.ones((4, 4)), np.zeros((4, 4))], + ext_name_list=["data", "noise_map"], + ) + fitsable.write_hdu_list(hdu_list, file_path=tmp_path / "dataset.fits") + + assert _stamp(tmp_path / "dataset.fits", hdu=0) is True + assert _stamp(tmp_path / "dataset.fits", hdu=1) is True + + +def test__write_hdu_list__stamps_an_hdu_list_built_elsewhere(tmp_path, monkeypatch): + # write_hdu_list is the terminal write, so it must stamp even an HDUList + # this module did not construct -- `hdu_list_for_output_from` is publicly + # re-exported as `aa.hdu_list_for_output_from`, so callers can and do build + # their own. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + hdu_list = fits.HDUList([fits.PrimaryHDU(np.ones((4, 4)))]) + fitsable.write_hdu_list(hdu_list, file_path=tmp_path / "raw.fits") + + assert _stamp(tmp_path / "raw.fits") is True + + +def test__stamp_does_not_disturb_header_dict_entries(tmp_path, monkeypatch): + # The stamp rides alongside the mask's PIXSCAY/PIXSCAX/ORIGINY/ORIGINX + # cards; it must not displace or overwrite any of them. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + + fitsable.output_to_fits( + np.ones((4, 4)), + file_path=tmp_path / "h.fits", + header_dict={"PIXSCAY": 0.5, "PIXSCAX": 0.5, "ORIGINY": 0.0, "ORIGINX": 0.0}, + ) + + header = fits.open(tmp_path / "h.fits")[0].header + assert header["PIXSCAY"] == 0.5 + assert header["PIXSCAX"] == 0.5 + assert header[KEY] is True + + +def test__stamp_is_idempotent_across_both_funnels(tmp_path, monkeypatch): + # hdu_list_for_output_from and write_hdu_list BOTH stamp, and output_to_fits + # goes through both. Assignment (not append) keeps that from duplicating the + # card -- a duplicate would make header[KEY] ambiguous. + monkeypatch.setenv("PYAUTO_SMALL_DATASETS", "1") + fitsable.output_to_fits(np.ones((4, 4)), file_path=tmp_path / "once.fits") + + header = fits.open(tmp_path / "once.fits")[0].header + assert len([c for c in header.cards if c.keyword == KEY]) == 1 From 553327ff03812886c4c6e522c408ee09d469775f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Aug 2026 15:04:15 +0000 Subject: [PATCH 2/5] docs: pin the 8-char ceiling on the regime stamp keyword A 9-character keyword is silently promoted to HIERARCH by astropy rather than raising, and header.get() by the short name then returns None. Readers treat None as unknown and fall back to the shape heuristic, so an over-long key would not fail loudly -- it would quietly un-fix the interferometer case the stamp exists for. Comment plus a test that pins the ceiling. --- autonerves/fitsable.py | 8 ++++++++ test_autonerves/test_fitsable.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/autonerves/fitsable.py b/autonerves/fitsable.py index 4bed148..a6b8979 100644 --- a/autonerves/fitsable.py +++ b/autonerves/fitsable.py @@ -14,6 +14,14 @@ from typing import Dict, Optional, Union, List +# Exactly 8 characters, and that ceiling is load-bearing rather than stylistic. +# A 9-character keyword does not raise here: astropy silently promotes it to a +# HIERARCH card, which `header.get("...")` by the short name then misses, +# returning None. The reader in ``autoarray.util.dataset_util`` treats None as +# "unknown regime" and falls back to its shape heuristic -- so an over-long key +# would not fail loudly, it would quietly un-fix the interferometer case this +# stamp exists for. Any rename must stay within 8 chars, and is a wire-format +# change (the reader duplicates this literal), not a refactor. SMALL_DATASETS_HEADER_KEY = "SMALLDAT" SMALL_DATASETS_HEADER_COMMENT = "PYAUTO_SMALL_DATASETS active at write time" diff --git a/test_autonerves/test_fitsable.py b/test_autonerves/test_fitsable.py index 424f86f..27f78f7 100644 --- a/test_autonerves/test_fitsable.py +++ b/test_autonerves/test_fitsable.py @@ -212,3 +212,13 @@ def test__stamp_is_idempotent_across_both_funnels(tmp_path, monkeypatch): header = fits.open(tmp_path / "once.fits")[0].header assert len([c for c in header.cards if c.keyword == KEY]) == 1 + + +def test__stamp_key_stays_within_the_fits_standard_card_limit(): + # Not stylistic. A 9-char keyword is silently promoted to a HIERARCH card + # by astropy rather than raising, and header.get() by the short name then + # returns None -- which readers treat as "unknown regime" and fall back to + # the shape heuristic. An over-long key would therefore not fail loudly, it + # would quietly un-fix the interferometer case. Pin the ceiling. + assert len(KEY) <= 8 + assert KEY == KEY.upper() From e582e52d09300c4137a541c4f63f11a0691406af Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Aug 2026 15:08:05 +0000 Subject: [PATCH 3/5] test: refresh the array output fixture for the regime stamp test_autonerves/files/array_out.fits is a test write target that the suite rewrites on every run, so its committed bytes must match what the tests now produce. Verified on clean main that the tree stays clean there, i.e. this dirtying is introduced by the stamp rather than pre-existing. --- test_autonerves/files/array_out.fits | Bin 5760 -> 5760 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test_autonerves/files/array_out.fits b/test_autonerves/files/array_out.fits index f5e4cb4bc8d10eeb8b2afb04a86c5894d8fe9e7e..85cddd500dfc4f577dcb3ba7422da00b1caf6067 100644 GIT binary patch delta 77 zcmZqBZP4BDfRQuU*U`tv#WBQoVxx$jn}WVVK%`@6h<`j-JRT_F80;DntdN*ol3A9j ckXWKnUX)pqs!)=do4WZnqbTQQ2W|&$0J%#Tpa1{> delta 21 ccmZqBZP4BDfN`?{lK|)B0H%P=4%`mh08pX^XaE2J From 95b7c36f37370dfda9e740dea099e6f90b075fa9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Aug 2026 15:36:43 +0000 Subject: [PATCH 4/5] docs: stop claiming the stamp reaches every FITS in existence It reaches every FITS the PyAuto libraries author -- all 18 library write sites build their HDUList through this module even when they call hdu_list.writeto themselves. It does not reach HDULists assembled by hand elsewhere (autofit/aggregator/summary/aggregate_fits.py) or workspace scripts using raw astropy (the lenstool converter in autolens_workspace). Those read as absent, which is the safe direction and exactly why absence must never mean full. --- autonerves/fitsable.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/autonerves/fitsable.py b/autonerves/fitsable.py index a6b8979..a1ae1f8 100644 --- a/autonerves/fitsable.py +++ b/autonerves/fitsable.py @@ -47,7 +47,18 @@ def stamp_small_datasets_regime(header): capped run writes identical ``NAXIS`` with different values and trips no assertion at all. - The card is written on **every** FITS the stack writes, in both regimes: + The card is written in both regimes on every FITS written through this + module -- which is every FITS the PyAuto libraries author, because all 18 + library write sites build their HDUList here even when they call + ``hdu_list.writeto`` themselves. It is **not** universal, and readers must + not assume it is: PyAutoFit's aggregator assembles some HDULists by hand + (``autofit/aggregator/summary/aggregate_fits.py``), and workspace scripts + that use raw astropy directly (e.g. the lenstool converter in + autolens_workspace) write unstamped files. Those read as absent, i.e. + unknown -- which is the safe direction, and exactly why absence must never + be read as "full resolution". + + The three states: - ``SMALLDAT = T`` -- written under ``PYAUTO_SMALL_DATASETS=1`` (capped). - ``SMALLDAT = F`` -- written at full resolution. From 3e6e601ff46bf0057d9dfda94fc22dd9492496f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 22 Aug 2026 16:11:06 +0000 Subject: [PATCH 5/5] test: make the suite hermetic against the ambient small-datasets flag Every FITS the stack writes now carries a SMALLDAT card whose value tracks PYAUTO_SMALL_DATASETS at write time. Several tests write into TRACKED fixture paths -- a pre-existing pattern, 14 such files across this repo and PyAutoArray -- so the bytes those tests produce had become a function of the shell: running the suite with PYAUTO_SMALL_DATASETS=1 exported, which should_simulate's own docstring calls the default for most harness runs, passed but left the working tree dirty. Verified against fresh main worktrees that this dirtying is introduced by the stamp and is not pre-existing. An autouse fixture clearing the var restores the property the stamp took away -- test output is a function of the test, not of the environment -- in one place, rather than by rewriting every fixture-writing test in a PR about a header card. Tests that need a regime set it with monkeypatch.setenv in their body, which runs after the fixture and wins. No test depended on the ambient value. Verified: suite green and tree clean both with the var exported and unset. Found by three independent review lenses, each reproducing it separately. --- test_autonerves/conftest.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_autonerves/conftest.py b/test_autonerves/conftest.py index b52a728..816a492 100644 --- a/test_autonerves/conftest.py +++ b/test_autonerves/conftest.py @@ -24,3 +24,26 @@ def make_config(files_directory): files_directory / "config", files_directory / "default", ) + + +@pytest.fixture(autouse=True) +def _regime_independent_test_output(monkeypatch): + """ + Clear ``PYAUTO_SMALL_DATASETS`` for every test unless the test sets it. + + Since PyAutoNerves#153 every FITS the stack writes carries a ``SMALLDAT`` + card whose value tracks this env var at write time. Several tests write into + **tracked** fixture paths (14 of them across PyAutoArray and this repo -- + a pre-existing pattern), so without this the bytes those tests produce + depend on the ambient environment: run the suite in a shell exporting + ``PYAUTO_SMALL_DATASETS=1`` -- which ``should_simulate``'s own docstring calls + the default for most harness runs -- and the suite passes but leaves the + working tree dirty. + + Pinning it here restores the property the stamp took away, that test output + is a function of the test and not of the shell, and does so in one place + rather than by rewriting every fixture-writing test. Tests that need a + regime set it with ``monkeypatch.setenv`` in their body, which runs after + this fixture and wins. + """ + monkeypatch.delenv("PYAUTO_SMALL_DATASETS", raising=False)