From a3d3cd2fa3d8e1d0019516296fbc9a7015e8183c Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Wed, 28 Oct 2026 16:18:31 +0100 Subject: [PATCH 1/2] Skip the index store in mark_indexes_stale for unindexed arrays NDArray.__setitem__ calls mark_indexes_stale() after every write. Loading the store resolves the urlpath and stats every cached persistent path, and on NFS each stat() of a file with buffered writes first pushes them to the server. Return early when no cached store has indexes and the array's vlmeta has no index store. Co-Authored-By: Claude Opus 5.5 --- src/blosc2/indexing.py | 18 +++++++++++++++++ tests/ndarray/test_indexing.py | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/blosc2/indexing.py b/src/blosc2/indexing.py index 892ff36ea..da6ee128f 100644 --- a/src/blosc2/indexing.py +++ b/src/blosc2/indexing.py @@ -5097,7 +5097,25 @@ def get_index(array: blosc2.NDArray, field: str | None = None, name: str | None return Index(array, token) +def _may_have_indexes(array: blosc2.NDArray) -> bool: + """Cheap pre-check for :func:`mark_indexes_stale`, which runs after every write. + + Resolving a persistent array's store costs a ``realpath()`` and ``stat()`` calls on + its file, and on NFS each ``stat()`` of a file with buffered writes first pushes them + to the server: a synchronous round trip per ``__setitem__``. A file whose vlmeta has + no index store, in a process that caches no store with indexes either, has nothing + to mark. + """ + if not _is_persistent_array(array): + return True + if any(store.get("indexes") for store in _PERSISTENT_INDEXES.values()): + return True + return INDEXES_VLMETA_KEY in array.schunk.vlmeta + + def mark_indexes_stale(array: blosc2.NDArray) -> None: + if not _may_have_indexes(array): + return store = _load_store(array) if not store["indexes"]: return diff --git a/tests/ndarray/test_indexing.py b/tests/ndarray/test_indexing.py index 23aab884e..89beaf6de 100644 --- a/tests/ndarray/test_indexing.py +++ b/tests/ndarray/test_indexing.py @@ -1886,6 +1886,41 @@ def test_invalidation_on_mark_indexes_stale(tmp_path): assert indexing._HOT_CACHE_BYTES == 0 +def test_setitem_without_indexes_skips_index_store(tmp_path, monkeypatch): + # Loading the store costs realpath() and stat() calls on the file for every write, and on + # NFS each stat() first pushes the writes still buffered for that file to the server + _clear_caches() + urlpath = str(tmp_path / "plain.b2nd") + arr = blosc2.zeros(1_000, dtype=np.int64, chunks=(250,), blocks=(50,), urlpath=urlpath, mode="w") + monkeypatch.setattr( + indexing, + "_load_store", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("an unindexed array has no store")), + ) + arr[:3] = -1 + arr[10:20] = 7 + np.testing.assert_array_equal(arr[:3], np.full(3, -1, dtype=np.int64)) + np.testing.assert_array_equal(arr[10:20], np.full(10, 7, dtype=np.int64)) + + +def test_setitem_marks_on_disk_index_stale_with_cold_caches(tmp_path): + # The index is only known from the file's vlmeta here, not from any in-process cache + urlpath = str(tmp_path / "indexed.b2nd") + arr = blosc2.asarray( + np.arange(1_000, dtype=np.int64), chunks=(250,), blocks=(50,), urlpath=urlpath, mode="w" + ) + arr.create_index(kind=blosc2.IndexKind.FULL) + del arr + _clear_caches() + + arr = blosc2.open(urlpath, mode="a") + arr[:3] = -1 + assert arr.indexes[0]["stale"] is True + del arr + _clear_caches() + assert blosc2.open(urlpath, mode="r").indexes[0]["stale"] is True + + def test_invalidation_on_append(tmp_path): arr, _ = _make_persistent_array(tmp_path) _clear_caches() From 028e5ac54a342fa5a66fec3c5d6913aca6eae502 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Thu, 29 Oct 2026 07:38:38 +0100 Subject: [PATCH 2/2] Tidy the mark_indexes_stale pre-check and its tests Move _may_have_indexes next to the store helpers, iterate over a snapshot of the persistent store cache, and use plain helpers in the two tests. Co-Authored-By: Claude Opus 5.5 --- src/blosc2/indexing.py | 32 ++++++++++++++++---------------- tests/ndarray/test_indexing.py | 16 ++++++---------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/blosc2/indexing.py b/src/blosc2/indexing.py index da6ee128f..cc344d3cd 100644 --- a/src/blosc2/indexing.py +++ b/src/blosc2/indexing.py @@ -555,6 +555,22 @@ def _load_store(array: blosc2.NDArray) -> dict: return store +def _may_have_indexes(array: blosc2.NDArray) -> bool: + """Cheap pre-check for :func:`mark_indexes_stale`, which runs after every write. + + Resolving a persistent array's store costs ``realpath()`` and ``stat()`` calls on its + file, and on NFS each ``stat()`` of a file with buffered writes first pushes them to + the server: a synchronous round trip per ``__setitem__``. A file whose vlmeta has no + index store, in a process that caches no store with indexes either, has nothing to + mark. + """ + if not _is_persistent_array(array): + return True + if any(store.get("indexes") for store in tuple(_PERSISTENT_INDEXES.values())): + return True + return INDEXES_VLMETA_KEY in array.schunk.vlmeta + + def _save_store(array: blosc2.NDArray, store: dict) -> None: store.setdefault("version", INDEX_FORMAT_VERSION) store.setdefault("indexes", {}) @@ -5097,22 +5113,6 @@ def get_index(array: blosc2.NDArray, field: str | None = None, name: str | None return Index(array, token) -def _may_have_indexes(array: blosc2.NDArray) -> bool: - """Cheap pre-check for :func:`mark_indexes_stale`, which runs after every write. - - Resolving a persistent array's store costs a ``realpath()`` and ``stat()`` calls on - its file, and on NFS each ``stat()`` of a file with buffered writes first pushes them - to the server: a synchronous round trip per ``__setitem__``. A file whose vlmeta has - no index store, in a process that caches no store with indexes either, has nothing - to mark. - """ - if not _is_persistent_array(array): - return True - if any(store.get("indexes") for store in _PERSISTENT_INDEXES.values()): - return True - return INDEXES_VLMETA_KEY in array.schunk.vlmeta - - def mark_indexes_stale(array: blosc2.NDArray) -> None: if not _may_have_indexes(array): return diff --git a/tests/ndarray/test_indexing.py b/tests/ndarray/test_indexing.py index 89beaf6de..3495b50d4 100644 --- a/tests/ndarray/test_indexing.py +++ b/tests/ndarray/test_indexing.py @@ -1892,11 +1892,11 @@ def test_setitem_without_indexes_skips_index_store(tmp_path, monkeypatch): _clear_caches() urlpath = str(tmp_path / "plain.b2nd") arr = blosc2.zeros(1_000, dtype=np.int64, chunks=(250,), blocks=(50,), urlpath=urlpath, mode="w") - monkeypatch.setattr( - indexing, - "_load_store", - lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("an unindexed array has no store")), - ) + + def fail_load_store(array): + raise AssertionError("an unindexed array has no store to load") + + monkeypatch.setattr(indexing, "_load_store", fail_load_store) arr[:3] = -1 arr[10:20] = 7 np.testing.assert_array_equal(arr[:3], np.full(3, -1, dtype=np.int64)) @@ -1905,11 +1905,7 @@ def test_setitem_without_indexes_skips_index_store(tmp_path, monkeypatch): def test_setitem_marks_on_disk_index_stale_with_cold_caches(tmp_path): # The index is only known from the file's vlmeta here, not from any in-process cache - urlpath = str(tmp_path / "indexed.b2nd") - arr = blosc2.asarray( - np.arange(1_000, dtype=np.int64), chunks=(250,), blocks=(50,), urlpath=urlpath, mode="w" - ) - arr.create_index(kind=blosc2.IndexKind.FULL) + arr, urlpath = _make_scalar_persistent_array(tmp_path) del arr _clear_caches()