From bc2051b3e459c35a0cc5e198c1a6f76b2bad5bce Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Wed, 4 Nov 2026 20:34:51 +0100 Subject: [PATCH 1/2] Add NDArray.read_into(): decompress a selection into an existing array arr[key] always returns a new array, so code that needs the data in an existing array (the interior of a padded patch, one item of a batch, a pinned or reused buffer) pays for a temporary, its page faults and one more copy. read_into(out, key) decompresses arr[key] straight into `out`, a NumPy array of the selection's shape. For basic keys (integers, slices with step 1, Ellipsis) and `out` of the array's dtype whose last axis is contiguous and whose other strides are whole multiples of the next one, i.e. any C-contiguous array or box of one, also with steps in the outer axes, the data goes straight to `out` through b2nd_get_slice_cbuffer_into() and nothing outside `out` is touched; the GIL is released meanwhile as in the other slice reads. Any other layout, key or dtype (`casting` as in numpy.copyto) goes through arr[key], so every writeable NumPy array works. As in the fast path of __getitem__, basic keys are resolved in Cython against the cached shape, and the read is retried if it finds that another handle changed the shape (SWMR), so small reads cost no more per call than arr[key]. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SwMZj7wsgYdoCR5uvYPwEo --- doc/reference/ndarray.rst | 2 + src/blosc2/blosc2_ext.pyx | 146 ++++++++++++-- src/blosc2/ndarray.py | 72 +++++++ tests/ndarray/test_read_into.py | 348 ++++++++++++++++++++++++++++++++ 4 files changed, 555 insertions(+), 13 deletions(-) create mode 100644 tests/ndarray/test_read_into.py diff --git a/doc/reference/ndarray.rst b/doc/reference/ndarray.rst index 8c492fce7..01821040e 100644 --- a/doc/reference/ndarray.rst +++ b/doc/reference/ndarray.rst @@ -25,6 +25,7 @@ It is a direct alias for ``array.vlmeta`` and uses the same persistent storage. __len__ __getitem__ __setitem__ + read_into take Utility Methods @@ -34,6 +35,7 @@ It is a direct alias for ``array.vlmeta`` and uses the same persistent storage. .. automethod:: __len__ .. automethod:: __getitem__ .. automethod:: __setitem__ + .. automethod:: read_into .. automethod:: take Constructors diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index f59628236..2910c3455 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -648,6 +648,9 @@ cdef extern from "b2nd.h": int b2nd_get_slice_cbuffer(b2nd_array_t *array, int64_t *start, int64_t *stop, void *buffer, int64_t *buffershape, int64_t buffersize) nogil + int b2nd_get_slice_cbuffer_into(b2nd_array_t *array, + int64_t *start, int64_t *stop, + void *buffer, int64_t *buffershape, int64_t buffersize) nogil int b2nd_set_slice_cbuffer(void *buffer, int64_t *buffershape, int64_t buffersize, int64_t *start, int64_t *stop, b2nd_array_t *array) int b2nd_get_slice(b2nd_context_t *ctx, b2nd_array_t **array, b2nd_array_t *src, const int64_t *start, @@ -4386,6 +4389,73 @@ cdef class NDArray: _check_rc(rc, "Error while getting the buffer") return arr + def get_basic_into(self, np.ndarray out, key): + """NDArray.read_into() for a basic index: decompress ``self[key]`` into the NumPy array + `out` and return it, or return None if *key* is not a basic index. + + The data goes straight into `out` if it has the array's dtype and is a box of a C-order + buffer (see _box_layout()); otherwise into a new C-contiguous array, which is returned + instead for the caller to copy into `out`. Keys are as in get_basic_numpy(), but without + None; any other key, and any key that is out of range, gives None, so that the caller can + use the general indexing path, which also raises the appropriate exception. A selection + of another shape than `out` raises ValueError. + """ + cdef tuple items = key if type(key) is tuple else (key,) + cdef Py_ssize_t nkey = len(items), i + cdef char[_MAX_BASIC_KEY] kinds + cdef int64_t[_MAX_BASIC_KEY] ints + cdef Py_ssize_t[_MAX_BASIC_KEY] starts, stops + cdef Py_ssize_t nidx = self._classify_basic_key(items, kinds, ints, starts, stops) + if nidx < 0: + return None + for i in range(nkey): + if kinds[i] == _KEY_NEWAXIS: + return None + + cdef int64_t[B2ND_MAX_DIM] start_, stop_, buffershape_ + cdef c_bool[B2ND_MAX_DIM] int_dims + cdef int64_t tick, buffersize = 0 + cdef int rc = 0, attempt + cdef c_bool direct = False + target = None + with self._handle_lock: + # As in get_basic_numpy(): a key resolved against the cached shape is right if the read + # does not find shape changes made by another handle (SWMR); otherwise follow them and + # try again. Also follow them first if the selection does not have out's shape (they + # may be why) or is empty (there is no read then). + for attempt in range(_BASIC_READ_ATTEMPTS): + if attempt > 0: + _check_rc(b2nd_refresh(self.array), "Error while refreshing the array") + tick = self.array.last_tick + out_shape = self._resolve_basic_key(nkey, kinds, ints, starts, stops, nidx, start_, stop_, + int_dims) + if out_shape is None: + if attempt > 0: + return None + continue + if tuple(out_shape) != ( out).shape: + if attempt > 0: + raise ValueError(f"out has shape {( out).shape}, but the selection has " + f"shape {tuple(out_shape)}") + continue + if np.PyArray_SIZE(out) == 0: + if attempt > 0: + return out + continue + if target is None: + direct = out.dtype == self.dtype and self._box_layout(out, int_dims, buffershape_, + &buffersize) + target = out if direct else np.empty(out_shape, dtype=self.dtype) + if direct: + rc = self._read_slice(start_, stop_, np.PyArray_DATA(out), buffershape_, buffersize, True) + else: + rc = self._get_slice_cbuffer(target, start_, stop_) + if rc >= 0 and self.array.last_tick == tick: + return target + # Out of attempts: as the general path, return what the last read gave + _check_rc(rc, "Error while getting the slice") + return target + cdef _resolve_basic_key(self, Py_ssize_t nkey, const char *kinds, const int64_t *ints, const Py_ssize_t *starts, const Py_ssize_t *stops, Py_ssize_t nidx, int64_t *start_, int64_t *stop_, c_bool *int_dims=NULL): @@ -4449,7 +4519,6 @@ cdef class NDArray: cdef int _get_slice_cbuffer(self, arr, int64_t *start_, int64_t *stop_) except? -9999: # The caller holds self._handle_lock; returns the rc of b2nd_get_slice_cbuffer() - global _nogil_reads cdef int64_t[B2ND_MAX_DIM] buffershape_ cdef int i for i in range(self.array.ndim): @@ -4459,22 +4528,73 @@ cdef class NDArray: cdef int rc = 0 PyObject_GetBuffer(arr, &view, PyBUF_SIMPLE) try: - # Decompress without the GIL, so that threads reading other arrays (or other handles - # of the same file) run in parallel; everything else that uses this super-chunk - # holds the lock meanwhile. - if _can_release_gil(self.array.sc): - _nogil_reads += 1 - with nogil: - rc = b2nd_get_slice_cbuffer(self.array, start_, stop_, - view.buf, buffershape_, view.len) - _nogil_reads -= 1 - else: - rc = b2nd_get_slice_cbuffer(self.array, start_, stop_, - view.buf, buffershape_, view.len) + rc = self._read_slice(start_, stop_, view.buf, buffershape_, view.len, False) finally: PyBuffer_Release(&view) return rc + cdef int _read_slice(self, int64_t *start_, int64_t *stop_, void *buf, int64_t *buffershape_, + int64_t buffersize, c_bool into) except? -9999: + """b2nd_get_slice_cbuffer(), or b2nd_get_slice_cbuffer_into() if `into`; returns its rc. + The caller holds self._handle_lock. The decompression runs without the GIL when no Python + callback can run on the way, so that threads reading other arrays (or other handles of the + same file) run in parallel; everything else that uses this super-chunk holds the lock + meanwhile.""" + global _nogil_reads + cdef int rc + if _can_release_gil(self.array.sc): + _nogil_reads += 1 + with nogil: + if into: + rc = b2nd_get_slice_cbuffer_into(self.array, start_, stop_, buf, buffershape_, buffersize) + else: + rc = b2nd_get_slice_cbuffer(self.array, start_, stop_, buf, buffershape_, buffersize) + _nogil_reads -= 1 + elif into: + rc = b2nd_get_slice_cbuffer_into(self.array, start_, stop_, buf, buffershape_, buffersize) + else: + rc = b2nd_get_slice_cbuffer(self.array, start_, stop_, buf, buffershape_, buffersize) + return rc + + cdef c_bool _box_layout(self, np.ndarray out, const c_bool *int_dims, int64_t *buffershape_, + int64_t *buffersize): + """Whether `out`, of the shape of a selection whose integer-indexed dimensions are + `int_dims`, is a box of a C-order buffer that starts at out's first item; if so, fill that + buffer's shape and its size in bytes from there (for b2nd_get_slice_cbuffer_into()). + Integer-indexed dimensions and the others of length 1 are size-1 axes of the box, whose + strides do not matter. Of the longer ones, the innermost must be contiguous, and every + other one's stride a whole multiple of the next one's that spans at least that one's + length.""" + cdef np.npy_intp *shape = np.PyArray_DIMS(out) + cdef np.npy_intp *strides = np.PyArray_STRIDES(out) + cdef int64_t itemsize = np.PyArray_ITEMSIZE(out) + cdef int64_t[B2ND_MAX_DIM] lens, steps + cdef int64_t size = itemsize + cdef int ndim = self.array.ndim, d, j = 0, inner = -1 + for d in range(ndim): + if int_dims[d]: + lens[d] = 1 + steps[d] = 0 + else: + lens[d] = shape[j] + steps[d] = strides[j] + j += 1 + buffershape_[d] = lens[d] + for d in range(ndim - 1, -1, -1): # the long dimensions, innermost first + if lens[d] <= 1: + continue + if inner < 0: + if steps[d] != itemsize or itemsize <= 0: + return False + elif steps[d] % steps[inner] != 0 or steps[d] // steps[inner] < lens[inner]: + return False + else: + buffershape_[inner] = steps[d] // steps[inner] + size += (lens[d] - 1) * steps[d] + inner = d + buffersize[0] = size + return True + @_handle_locked def get_1d_span_numpy(self, arr, int64_t nchunk, int32_t start, int32_t nitems): if self.ndim != 1: diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index 2b517307b..23bad401a 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -4967,6 +4967,78 @@ def _remember_read(self, key, nparr: np.ndarray) -> None: self._last_read.clear() self._last_read[make_key_hashable(key)] = nparr + def read_into(self, out: np.ndarray, key: Any = None, casting: str = "same_kind") -> np.ndarray: + """Decompress ``self[key]`` into the existing NumPy array ``out``. + + This is ``out[...] = self[key]`` without the temporary array: the data is decompressed + straight to its place in ``out``, which can be a view into a larger array, e.g. the + interior of a padded array, one item of a batch or one channel of a multi-channel array. + Nothing of the larger array outside ``out`` is touched. Reusing ``out`` across calls also + saves the page faults of a fresh result array every time. + + Parameters + ---------- + out: np.ndarray + Writeable NumPy array with the shape of ``self[key]``. Contiguous or not: any layout + works. Data goes straight into ``out`` (no temporary) when ``out`` has ``self.dtype`` and + its last axis is contiguous, with any strides in the other axes that are whole + multiples of the next axis' stride. That covers C-contiguous arrays and every box of one, + including views with steps in all but the last axis. Other layouts (e.g. Fortran order, + transposed views, steps along the last axis) and other dtypes go through one temporary. + For a CPU ``torch.Tensor`` ``t``, pass ``t.numpy()``. + key: int, slice, Ellipsis or tuple of them, or any key that :meth:`__getitem__` accepts + The selection to read, as in ``self[key]``; None (default) reads the whole array. + Integers, slices with step 1 and an Ellipsis decompress straight into ``out``; other + keys (steps, integer or boolean arrays, ``None``) go through ``self[key]``. + casting: str + How ``self.dtype`` may be converted to ``out.dtype``, as in :func:`numpy.copyto`. + Default ``"same_kind"``. + + Returns + ------- + out: np.ndarray + The ``out`` array. + + Examples + -------- + >>> import numpy as np + >>> import blosc2 + >>> a = blosc2.arange(12, dtype=np.int32, shape=(3, 4)) + >>> padded = np.full((5, 6), -1, dtype=np.int32) + >>> _ = a.read_into(padded[1:4, 1:5]) # into the interior of a padded array + >>> padded + array([[-1, -1, -1, -1, -1, -1], + [-1, 0, 1, 2, 3, -1], + [-1, 4, 5, 6, 7, -1], + [-1, 8, 9, 10, 11, -1], + [-1, -1, -1, -1, -1, -1]], dtype=int32) + >>> batch = np.empty((2, 2, 4), dtype=np.int32) + >>> _ = a.read_into(batch[1], (slice(1, 3), slice(None))) # into one item of a batch + >>> batch[1] + array([[ 4, 5, 6, 7], + [ 8, 9, 10, 11]], dtype=int32) + """ + if not isinstance(out, np.ndarray): + raise TypeError(f"out must be a NumPy array, not {type(out).__name__}") + if not out.flags.writeable: + raise ValueError("out is read-only") + if key is None: + key = () + # Integers, slices with step 1 and an Ellipsis: straight into out, or into a temporary for + # another dtype or layout (None: another key, see get_basic_into) + value = self.get_basic_into(out, key) + if value is out: + return out + if value is None: # steps, arrays, newaxis, masks, fields...: through __getitem__ + value = self[key] + if isinstance(value, (blosc2.LazyExpr, NDField)): + value = value[:] + value = np.asarray(value) + if out.shape != value.shape: + raise ValueError(f"out has shape {out.shape}, but the selection has shape {value.shape}") + np.copyto(out, value, casting=casting) + return out + def __setitem__( # noqa: C901 self, key: int | slice | Sequence[slice | int | np.bool_ | np.ndarray[int | np.bool_] | None] | None, diff --git a/tests/ndarray/test_read_into.py b/tests/ndarray/test_read_into.py new file mode 100644 index 000000000..2c7bcaaae --- /dev/null +++ b/tests/ndarray/test_read_into.py @@ -0,0 +1,348 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +import threading + +import numpy as np +import pytest + +import blosc2 + +SENTINEL = 77 + + +@pytest.fixture(params=["memory", "contiguous", "sparse", "mmap"]) +def storage(request, tmp_path): + kind = request.param + + def make(a, chunks, blocks): + if kind == "memory": + return blosc2.asarray(a, chunks=chunks, blocks=blocks) + urlpath = str(tmp_path / ("a.b2nd" if kind != "sparse" else "a.b2frame")) + blosc2.asarray( + a, chunks=chunks, blocks=blocks, urlpath=urlpath, contiguous=kind != "sparse", mode="w" + ) + return blosc2.open(urlpath, mode="r", **({"mmap_mode": "r"} if kind == "mmap" else {})) + + return make + + +def source(shape, dtype, zero_half=True): + a = (np.arange(np.prod(shape)).reshape(shape) * 7 % 1000).astype(dtype) + if zero_half: # some all-zero chunks + a[: shape[0] // 2] = 0 + return a + + +def no_getitem(monkeypatch): + """Make NDArray.__getitem__ fail, to check that read_into decompresses straight into out""" + + def fail(self, key): + raise AssertionError("read_into went through __getitem__") + + monkeypatch.setattr(blosc2.NDArray, "__getitem__", fail) + + +KEYS = [ + None, + (), + Ellipsis, + 1, + -1, + (slice(2, 9), slice(1, 6)), + (slice(None), 3, slice(-5, None)), + (Ellipsis, slice(0, 4)), + (2, Ellipsis), + (slice(3, 100), slice(-100, 2)), # clipped like NumPy + (np.int64(4), slice(None), np.int32(-2)), +] + + +def read_and_check(b, a, key, monkeypatch): + expected = a[key] if key is not None else a + out = np.full(expected.shape, SENTINEL, dtype=a.dtype) + no_getitem(monkeypatch) + assert b.read_into(out, key) is out + np.testing.assert_array_equal(out, expected) + + +@pytest.mark.parametrize("key", KEYS) +def test_keys(storage, key, monkeypatch): + a = source((12, 9, 7), np.float32) + read_and_check(storage(a, (5, 4, 4), (2, 2, 3)), a, key, monkeypatch) + + +@pytest.mark.parametrize("dtype", [np.float32, np.int16, np.uint8, np.float64, np.complex64, np.bool_]) +def test_dtypes(storage, dtype, monkeypatch): + a = source((12, 9, 7), dtype) + read_and_check(storage(a, (5, 4, 4), (2, 2, 3)), a, (slice(1, 11), 4, slice(None)), monkeypatch) + + +@pytest.mark.parametrize( + ("key", "pad"), + [ + ((slice(0, 12), slice(0, 9), slice(0, 7)), ((2, 1), (0, 3), (1, 1))), + ((slice(5, 12), slice(2, 9), slice(3, 4)), ((0, 0), (4, 4), (2, 0))), + ((slice(1, 2), slice(0, 9), slice(0, 7)), ((3, 3), (1, 0), (0, 2))), + ], +) +def test_interior_of_padded_array(storage, key, pad, monkeypatch): + a = source((12, 9, 7), np.float32) + b = storage(a, (5, 4, 4), (2, 2, 3)) + inner = tuple(s.stop - s.start for s in key) + padded = np.full( + [n + lo + hi for n, (lo, hi) in zip(inner, pad, strict=True)], SENTINEL, dtype=np.float32 + ) + view = padded[tuple(slice(lo, lo + n) for n, (lo, _) in zip(inner, pad, strict=True))] + expected = np.full_like(padded, SENTINEL) + expected[tuple(slice(lo, lo + n) for n, (lo, _) in zip(inner, pad, strict=True))] = a[key] + no_getitem(monkeypatch) + b.read_into(view, key) + np.testing.assert_array_equal(padded, expected) # nothing outside the view changed + + +def test_batch_items_and_channels(storage, monkeypatch): + a = source((4, 10, 11), np.int16) + b = storage(a, (1, 6, 5), (1, 3, 5)) + batch = np.full((3, 2, 6, 11), SENTINEL, dtype=np.int16) + no_getitem(monkeypatch) + for j in range(3): + b.read_into(batch[j], (slice(j, j + 2), slice(2, 8))) # one item of a batch + for j in range(3): + np.testing.assert_array_equal(batch[j], a[j : j + 2, 2:8]) + channels = np.full((6, 4, 10, 11), SENTINEL, dtype=np.int16) + b.read_into(channels[3, 1], 2) # one channel of a multi-channel buffer + np.testing.assert_array_equal(channels[3, 1], a[2]) + assert (np.delete(channels.reshape(24, 10, 11), 3 * 4 + 1, axis=0) == SENTINEL).all() + + +@pytest.mark.parametrize( + "make_view", + [ + lambda p: p[::2, :6, :5], # steps in the outer axes are fine + lambda p: p[1:9, ::3, :5], + lambda p: p[:8, :6, 2:][:, 1:2, :5], # size-1 axis + lambda p: p[3:4, :, :5][:, ::2], + ], +) +def test_strided_views_without_temporary(storage, make_view, monkeypatch): + a = source((8, 6, 5), np.float64) + b = storage(a, (4, 3, 5), (2, 3, 5)) + parent = np.full((16, 12, 9), SENTINEL, dtype=np.float64) + view = make_view(parent) + key = tuple(slice(0, n) for n in view.shape) + expected = parent.copy() + make_view(expected)[...] = a[key] + no_getitem(monkeypatch) + b.read_into(view, key) + np.testing.assert_array_equal(parent, expected) + + +@pytest.mark.parametrize( + "make_out", + [ + lambda shape: np.full(shape, SENTINEL, dtype=np.float32, order="F"), + lambda shape: np.full(shape[::-1], SENTINEL, dtype=np.float32).T, # transposed + lambda shape: np.full((*shape[:-1], 2 * shape[-1]), SENTINEL, dtype=np.float32)[ + ..., ::2 + ], # last-axis step + lambda shape: np.full(shape, SENTINEL, dtype=np.float32)[::-1], # negative stride + ], +) +def test_other_layouts(storage, make_out): + a = source((9, 8, 7), np.float32) + b = storage(a, (4, 4, 4), (2, 2, 2)) + out = make_out((5, 8, 6)) + b.read_into(out, (slice(2, 7), slice(None), slice(1, 7))) + np.testing.assert_array_equal(out, a[2:7, :, 1:7]) + + +def test_other_keys_and_dtypes(): + a = source((10, 8), np.int16, zero_half=False) + b = blosc2.asarray(a, chunks=(4, 4), blocks=(2, 2)) + out = np.empty((5, 4), dtype=np.int16) + b.read_into(out, (slice(0, 10, 2), slice(1, 8, 2))) # steps + np.testing.assert_array_equal(out, a[::2, 1::2]) + out = np.empty((3, 8), dtype=np.int16) + b.read_into(out, [1, 4, 7]) # integer array + np.testing.assert_array_equal(out, a[[1, 4, 7]]) + out = np.empty((2, 8), dtype=np.int16) + b.read_into(out, (blosc2.asarray(np.array([1, 3])), slice(None))) # a blosc2 index array in a tuple + np.testing.assert_array_equal(out, a[[1, 3]]) + out = np.empty(int((a > 10).sum()), dtype=np.int16) + # a mask (lazy expression); its values come in chunk order, as from b[b > 10][:] + b.read_into(out, b > 10) + np.testing.assert_array_equal(out, b[b > 10][:]) + s = blosc2.asarray(np.array([(1.5, 2), (3.5, 4)], dtype=[("x", "f4"), ("y", "i4")])) + out = np.empty(2, dtype=np.float32) + s.read_into(out, "x") # a field + np.testing.assert_array_equal(out, [1.5, 3.5]) + out = np.empty((10, 8), dtype=np.float32) + b.read_into(out) # int16 into float32 (same_kind) + np.testing.assert_array_equal(out, a.astype(np.float32)) + out = np.empty((10, 8), dtype=np.uint8) + with pytest.raises(TypeError): + b.read_into(out) # int16 into uint8 is not same_kind + b.read_into(out, casting="unsafe") + np.testing.assert_array_equal(out, a.astype(np.uint8)) + + +def test_empty_and_0d(): + a = source((6, 5), np.float32) + b = blosc2.asarray(a, chunks=(4, 4), blocks=(2, 2)) + out = np.full((0, 5), SENTINEL, dtype=np.float32) + assert b.read_into(out, slice(3, 3)) is out + out = np.full((), SENTINEL, dtype=np.float32) + b.read_into(out, (4, 2)) + assert out == a[4, 2] + z = blosc2.asarray(np.float32(3.5)) + out = np.zeros((), dtype=np.float32) + z.read_into(out) + assert out == 3.5 + u = blosc2.uninit((), dtype=np.float32) # reads as zero, as u[()] does + out = np.full((), SENTINEL, dtype=np.float32) + u.read_into(out) + assert out == u[()] == 0 + + +def test_errors(): + a = source((6, 5), np.float32) + b = blosc2.asarray(a, chunks=(4, 4), blocks=(2, 2)) + with pytest.raises(ValueError, match="shape"): + b.read_into(np.empty((6, 4), dtype=np.float32)) + with pytest.raises(ValueError, match="shape"): + b.read_into(np.empty((3, 3), dtype=np.float32), (slice(0, 6, 2), slice(0, 4, 2))) + out = np.empty((6, 5), dtype=np.float32) + out.flags.writeable = False + with pytest.raises(ValueError, match="read-only"): + b.read_into(out) + with pytest.raises(TypeError, match="NumPy array"): + b.read_into([[0] * 5] * 6) + with pytest.raises(IndexError): + b.read_into(np.empty((), dtype=np.float32), (6, 0)) + with pytest.raises(IndexError): + b.read_into(np.empty((), dtype=np.float32), (0, 0, 0)) + + +@pytest.mark.parametrize( + "key", [None, 15, -1, slice(None), slice(-12, None), (Ellipsis, 3), (np.int64(19), 2), (slice(3, 3), 0)] +) +@pytest.mark.parametrize("order", ["C", "F"]) +def test_follows_resize(tmp_path, key, order): + """Keys are resolved against the shape another handle just set (SWMR), into C- and F-order out""" + urlpath = str(tmp_path / "keys.b2nd") + w = blosc2.asarray( + np.arange(100).reshape(10, 10), chunks=(4, 4), blocks=(2, 2), urlpath=urlpath, mode="w" + ) + r = blosc2.open(urlpath) + _ = r[0] + np_key = () if key is None else key + + def check(expected): + try: + want = expected[np_key] + except IndexError: + with pytest.raises(IndexError): + r.read_into(np.empty((10,), dtype=w.dtype), key) + else: + out = np.full(want.shape, SENTINEL, dtype=w.dtype, order=order) + np.testing.assert_array_equal(r.read_into(out, key), want) + + w.resize((20, 10)) + w[10:20] = np.arange(100, 200).reshape(10, 10) + check(np.arange(200).reshape(20, 10)) + w.resize((5, 10)) + check(np.arange(50).reshape(5, 10)) + + +def test_stale_out_shape_after_resize(tmp_path): + """An out of the shape the selection had before another handle resized the array""" + urlpath = str(tmp_path / "stale.b2nd") + w = blosc2.asarray( + np.arange(100).reshape(10, 10), chunks=(4, 4), blocks=(2, 2), urlpath=urlpath, mode="w" + ) + r = blosc2.open(urlpath) + _ = r[0] + w.resize((20, 10)) + with pytest.raises(ValueError, match="shape"): + r.read_into(np.empty((10, 10), dtype=w.dtype)) + out = np.empty((20, 10), dtype=w.dtype) + np.testing.assert_array_equal(r.read_into(out)[:10], np.arange(100).reshape(10, 10)) + + +@pytest.mark.skipif(blosc2.IS_WASM, reason="Pyodide cannot start threads") +def test_threads(tmp_path): + a = source((64, 32, 32), np.float32, zero_half=False) + urlpath = str(tmp_path / "t.b2nd") + blosc2.asarray(a, chunks=(16, 16, 16), blocks=(8, 8, 8), urlpath=urlpath, mode="w") + out = np.full((8, 66, 32, 32), SENTINEL, dtype=np.float32) + errors = [] + + def work(t): + try: + h = blosc2.open(urlpath, mode="r") # one handle per thread + for k in range(t, 64, 8): + h.read_into(out[t, 1 + k : 2 + k], (slice(k, k + 1),)) + except Exception as e: # pragma: no cover + errors.append(e) + + ts = [threading.Thread(target=work, args=(t,)) for t in range(8)] + for t in ts: + t.start() + for t in ts: + t.join() + assert not errors + for t in range(8): + for k in range(t, 64, 8): + np.testing.assert_array_equal(out[t, 1 + k], a[k]) + + +@pytest.mark.parametrize("seed", range(4)) +def test_random_views(seed): + """Random keys into random views of larger arrays (steps, size-1 axes, integer axes, + reversed axes, Fortran order): read_into equals NumPy and changes nothing outside out""" + rng = np.random.default_rng(seed) + for it in range(60): + ndim = int(rng.integers(1, 5)) + shape = tuple(int(rng.integers(1, 9)) for _ in range(ndim)) + chunks = tuple(int(rng.integers(1, s + 1)) for s in shape) + blocks = tuple(int(rng.integers(1, c + 1)) for c in chunks) + dtype = rng.choice([np.uint8, np.int16, np.float32, np.float64]) + a = (rng.random(shape) * 100).astype(dtype) # in range for every dtype (no invalid casts) + a[tuple(slice(0, s // 2) for s in shape)] = 0 + b = blosc2.asarray(a, chunks=chunks, blocks=blocks) + key = [] + for s in shape: + if rng.random() < 0.25: + key.append(int(rng.integers(-s, s))) + else: + lo = int(rng.integers(0, s + 1)) + key.append(slice(lo, int(rng.integers(lo, s + 1)))) + key = tuple(key) + expected = np.asarray(a[key]) + # a view with the result's shape into a parent with room around it + steps = [int(rng.integers(1, 3)) for _ in expected.shape] + pads = [int(rng.integers(0, 3)) for _ in expected.shape] + pshape = [ + n * st + p + int(rng.integers(0, 2)) + for n, st, p in zip(expected.shape, steps, pads, strict=True) + ] + order = "F" if rng.random() < 0.15 else "C" + reverse = expected.ndim > 0 and rng.random() < 0.15 + sel = tuple(slice(p, p + n * st, st) for n, st, p in zip(expected.shape, steps, pads, strict=True)) + + def view_of(arr, sel=sel, reverse=reverse): + v = arr[(*sel, Ellipsis)] # a view also for 0-d arrays + return v[::-1] if reverse else v + + parent = np.full(pshape, SENTINEL, dtype=dtype, order=order) + ref = parent.copy(order="K") + view_of(ref)[...] = expected + b.read_into(view_of(parent), key) + np.testing.assert_array_equal( + parent, ref, err_msg=f"{it} {shape} {chunks} {blocks} {key} {sel} {reverse}" + ) From 870287345942241eb9ceb665647b8013a7f89cb5 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Thu, 5 Nov 2026 13:13:01 +0100 Subject: [PATCH 2/2] Bundle c-blosc2 main e3ff64f2 for b2nd_get_slice_cbuffer_into() read_into() needs b2nd_get_slice_cbuffer_into() (c-blosc2 #845), which is on c-blosc2 main but not in a release yet. Pin the bundled c-blosc2 to the current main commit until the next release, and require at least 3.5.1 when building against a system c-blosc2 (3.5.0 lacks the function; main reports 3.5.1.dev, which pkg-config accepts). Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6990d1bcb..1eabdfda1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,9 +15,9 @@ endif() project(python-blosc2) # Update to the latest minimum C-Blosc2 version, and to the actual bundled one -set(BLOSC2_MIN_VERSION 3.5.0) -set(BLOSC2_BUNDLED_VERSION v3.5.0) -#set(BLOSC2_BUNDLED_VERSION a54e259aeb1c0e62d6bbea28934369d4831e4046) +set(BLOSC2_MIN_VERSION 3.5.1) +#set(BLOSC2_BUNDLED_VERSION v3.5.0) +set(BLOSC2_BUNDLED_VERSION e3ff64f261522803c6b9af7be913ef1c530cbc04) if(WIN32 AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") message(FATAL_ERROR "Windows builds require clang-cl. Set CC/CXX to clang-cl or configure CMake with -T ClangCL.")