Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions src/blosc2/indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,22 @@ def _load_store(array: blosc2.NDArray) -> dict:
return store


def _may_have_indexes(array: blosc2.NDArray) -> bool:
"""Cheap pre-check for :func:`mark_indexes_stale`, which runs after every write.

Resolving a persistent array's store costs ``realpath()`` and ``stat()`` calls on its
file, and on NFS each ``stat()`` of a file with buffered writes first pushes them to
the server: a synchronous round trip per ``__setitem__``. A file whose vlmeta has no
index store, in a process that caches no store with indexes either, has nothing to
mark.
"""
if not _is_persistent_array(array):
return True
if any(store.get("indexes") for store in tuple(_PERSISTENT_INDEXES.values())):
return True
return INDEXES_VLMETA_KEY in array.schunk.vlmeta


def _save_store(array: blosc2.NDArray, store: dict) -> None:
store.setdefault("version", INDEX_FORMAT_VERSION)
store.setdefault("indexes", {})
Expand Down Expand Up @@ -5098,6 +5114,8 @@ def get_index(array: blosc2.NDArray, field: str | None = None, name: str | None


def mark_indexes_stale(array: blosc2.NDArray) -> None:
if not _may_have_indexes(array):
return
store = _load_store(array)
if not store["indexes"]:
return
Expand Down
31 changes: 31 additions & 0 deletions tests/ndarray/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1886,6 +1886,37 @@ def test_invalidation_on_mark_indexes_stale(tmp_path):
assert indexing._HOT_CACHE_BYTES == 0


def test_setitem_without_indexes_skips_index_store(tmp_path, monkeypatch):
# Loading the store costs realpath() and stat() calls on the file for every write, and on
# NFS each stat() first pushes the writes still buffered for that file to the server
_clear_caches()
urlpath = str(tmp_path / "plain.b2nd")
arr = blosc2.zeros(1_000, dtype=np.int64, chunks=(250,), blocks=(50,), urlpath=urlpath, mode="w")

def fail_load_store(array):
raise AssertionError("an unindexed array has no store to load")

monkeypatch.setattr(indexing, "_load_store", fail_load_store)
arr[:3] = -1
arr[10:20] = 7
np.testing.assert_array_equal(arr[:3], np.full(3, -1, dtype=np.int64))
np.testing.assert_array_equal(arr[10:20], np.full(10, 7, dtype=np.int64))


def test_setitem_marks_on_disk_index_stale_with_cold_caches(tmp_path):
# The index is only known from the file's vlmeta here, not from any in-process cache
arr, urlpath = _make_scalar_persistent_array(tmp_path)
del arr
_clear_caches()

arr = blosc2.open(urlpath, mode="a")
arr[:3] = -1
assert arr.indexes[0]["stale"] is True
del arr
_clear_caches()
assert blosc2.open(urlpath, mode="r").indexes[0]["stale"] is True


def test_invalidation_on_append(tmp_path):
arr, _ = _make_persistent_array(tmp_path)
_clear_caches()
Expand Down
Loading