diff --git a/src/blosc2/indexing.py b/src/blosc2/indexing.py index 892ff36ea..cc344d3cd 100644 --- a/src/blosc2/indexing.py +++ b/src/blosc2/indexing.py @@ -555,6 +555,22 @@ def _load_store(array: blosc2.NDArray) -> dict: return store +def _may_have_indexes(array: blosc2.NDArray) -> bool: + """Cheap pre-check for :func:`mark_indexes_stale`, which runs after every write. + + Resolving a persistent array's store costs ``realpath()`` and ``stat()`` calls on its + file, and on NFS each ``stat()`` of a file with buffered writes first pushes them to + the server: a synchronous round trip per ``__setitem__``. A file whose vlmeta has no + index store, in a process that caches no store with indexes either, has nothing to + mark. + """ + if not _is_persistent_array(array): + return True + if any(store.get("indexes") for store in tuple(_PERSISTENT_INDEXES.values())): + return True + return INDEXES_VLMETA_KEY in array.schunk.vlmeta + + def _save_store(array: blosc2.NDArray, store: dict) -> None: store.setdefault("version", INDEX_FORMAT_VERSION) store.setdefault("indexes", {}) @@ -5098,6 +5114,8 @@ def get_index(array: blosc2.NDArray, field: str | None = None, name: str | None def mark_indexes_stale(array: blosc2.NDArray) -> None: + if not _may_have_indexes(array): + return store = _load_store(array) if not store["indexes"]: return diff --git a/tests/ndarray/test_indexing.py b/tests/ndarray/test_indexing.py index 23aab884e..3495b50d4 100644 --- a/tests/ndarray/test_indexing.py +++ b/tests/ndarray/test_indexing.py @@ -1886,6 +1886,37 @@ def test_invalidation_on_mark_indexes_stale(tmp_path): assert indexing._HOT_CACHE_BYTES == 0 +def test_setitem_without_indexes_skips_index_store(tmp_path, monkeypatch): + # Loading the store costs realpath() and stat() calls on the file for every write, and on + # NFS each stat() first pushes the writes still buffered for that file to the server + _clear_caches() + urlpath = str(tmp_path / "plain.b2nd") + arr = blosc2.zeros(1_000, dtype=np.int64, chunks=(250,), blocks=(50,), urlpath=urlpath, mode="w") + + def fail_load_store(array): + raise AssertionError("an unindexed array has no store to load") + + monkeypatch.setattr(indexing, "_load_store", fail_load_store) + arr[:3] = -1 + arr[10:20] = 7 + np.testing.assert_array_equal(arr[:3], np.full(3, -1, dtype=np.int64)) + np.testing.assert_array_equal(arr[10:20], np.full(10, 7, dtype=np.int64)) + + +def test_setitem_marks_on_disk_index_stale_with_cold_caches(tmp_path): + # The index is only known from the file's vlmeta here, not from any in-process cache + arr, urlpath = _make_scalar_persistent_array(tmp_path) + del arr + _clear_caches() + + arr = blosc2.open(urlpath, mode="a") + arr[:3] = -1 + assert arr.indexes[0]["stale"] is True + del arr + _clear_caches() + assert blosc2.open(urlpath, mode="r").indexes[0]["stale"] is True + + def test_invalidation_on_append(tmp_path): arr, _ = _make_persistent_array(tmp_path) _clear_caches()