Faster NDArray reads, writes, fills, serialization and expressions; fixes for crashes and wrong results

CMakeLists.txt

@@ -17,6 +17,9 @@ project(python-blosc2)
 # Update to the latest minimum C-Blosc2 version, and to the actual bundled one
 set(BLOSC2_MIN_VERSION 3.3.0)
 set(BLOSC2_BUNDLED_VERSION v3.3.5)
+# TEMPORARY: build against the companion c-blosc2 PR branch until it is merged and released
+set(BLOSC2_BUNDLED_REPOSITORY https://github.com/FabianIsensee/c-blosc2)
+set(BLOSC2_BUNDLED_VERSION 903bf73f002fd2b9298cd606a725dd31a88c5367)
 #set(BLOSC2_BUNDLED_VERSION a54e259aeb1c0e62d6bbea28934369d4831e4046)
 
 if(WIN32 AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
@@ -219,7 +222,7 @@ else()
     set(BLOSC_INSTALL ON)
     include(FetchContent)
     FetchContent_Declare(blosc2
-        GIT_REPOSITORY https://github.com/Blosc/c-blosc2
+        GIT_REPOSITORY ${BLOSC2_BUNDLED_REPOSITORY}
         GIT_TAG ${BLOSC2_BUNDLED_VERSION}
         # SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../c-blosc2
     )

RELEASE_NOTES.md

@@ -127,6 +127,63 @@ XXX version-specific blurb XXX
   plotting requires an explicit bounded row window. Client caches are separate
   from a shared cat2lite server cache; repository allowances are per root.
 
+### Faster NDArray indexing, writes and expressions
+
+- Basic keys (integers, unit-step slices, `...`, `None`) in
+  `NDArray.__getitem__` and `__setitem__` are resolved in Cython: `a[3, 5, 7]`
+  and row reads take ~30% less time, keys with NumPy integers up to 5x less.
+- Creating small arrays costs a third less (`zeros((100, 100))` 31 -> 20 µs);
+  `arange`, `linspace`, `eye`, `reshape` and lazy functions such as `clip` no
+  longer call `inspect.stack()` (`clip()` 84 -> 12 µs).
+- `a[region] = scalar` stores fully covered chunks as special chunks and writes
+  the rest in bounded pieces instead of building the region with `np.full()`
+  (`a[:] = 0.0` on 300 MB: 41 -> 0.2 ms, no extra memory). `a[region] = ndarray`
+  goes chunk by chunk too, copying compressed chunks when both arrays encode
+  them the same way (8x faster, a tenth of the memory). Arrays opened with
+  `locking=True` keep the single atomic write.
+- `asarray()`/`astype()` of an NDArray cast whole chunks in reused buffers
+  (`astype(np.uint8)` 2x faster).
+- `to_cframe()` writes the frame straight into the returned `bytes` (3.5x
+  faster, half the peak memory).
+- Lazy expressions: `expr[:]` evaluates straight into the NumPy result
+  (`(a + b)[:]` 2x faster), full `std()`/`var()` are two single-pass reductions
+  (3x faster), chunked reductions decompress into reused buffers, and
+  `where(cond, x, y)` with a scalar `x` or `y` runs in miniexpr.
+- `SChunk.decompress_chunk()` no longer reads the chunk from a frame twice.
+
+### Bug fixes
+
+- Fixed crashes of lazy expressions over in-memory frame-backed arrays
+  (`contiguous=True`, or `ndarray_from_cframe()` without a copy) and of the fast
+  `matmul` path with on-disk or frame-backed operands.
+- Expressions that the bundled miniexpr evaluates wrongly or crashes on now take
+  the numexpr path: reductions with constant subexpressions, true division and
+  two-argument float functions of integers or bools, `%` with negative
+  operands, constants compared with integers, some mixed-dtype expressions,
+  `bool + bool`, and grammar differences such as `-x ** 2`.
+- The result dtype of `hypot`, `copysign` and `nextafter` of integer
+  subexpressions and of true division involving bools follows NumPy (results
+  were truncated to integers).
+- `NDArray.__getitem__` accepts the keys NumPy accepts that raised or gave a
+  wrong shape: huge slice bounds, bool steps, 0-d integer arrays, scalar bools
+  (placed and broadcast as in NumPy) and `None` next to array indices.
+- `SChunk.update_special()` on an NDArray's super-chunk wrote chunks with the
+  wrong block size, so later reads of the array failed.
+- `update_data()`/`insert_data()` with `copy=False` leaked their chunk when the
+  write failed.
+- True positions of 1-D bool masks no longer run past the array length.
+- `blosc2.nthreads` and the `CParams`/`DParams` defaults follow the
+  `BLOSC_NTHREADS` environment variable, which C-Blosc2 always applied; importing
+  blosc2 no longer overrides `NUMEXPR_NUM_THREADS`.
+
+### Compatibility notes
+
+- Requires the next C-Blosc2 release, for `blosc2_schunk_to_buffer_alloc()` and
+  for its rule that a failed chunk write without a copy leaves the chunk with
+  the caller.
+- Keys with `None` return arrays that own their data (formerly views of a
+  temporary). `std()`/`var()` may differ from before in the last bits.
+
 ## Changes from 4.14.0 to 4.14.1
 
 Python-Blosc2 4.14.1 is a security and feature release introducing safe

doc/reference/ndarray.rst

@@ -14,7 +14,7 @@ It is a direct alias for ``array.vlmeta`` and uses the same persistent storage.
 .. autoclass:: NDArray
     :members:
     :inherited-members:
-    :exclude-members: get_slice, set_slice, get_slice_numpy, get_oindex_numpy, set_oindex_numpy
+    :exclude-members: get_slice, set_slice, get_slice_numpy, get_basic_numpy, get_basic_start_stop, get_oindex_numpy, set_oindex_numpy
     :member-order: groupwise
 
     :Special Methods:

src/blosc2/__init__.py

@@ -503,6 +503,8 @@ def _raise(exc):
 cpu_info = get_cpu_info()
 nthreads = ncores = cpu_info.get("count", 1)
 """Number of threads to be used in compression/decompression.
+
+It starts from the ``BLOSC_NTHREADS`` environment variable if that is set.
 """
 # Protection against too many threads
 nthreads = min(nthreads, 64)
@@ -515,14 +517,22 @@ def _raise(exc):
     # Experiments say that, when using a large number of threads, it is better to not use them all
     if nthreads > 16:
         nthreads -= nthreads // 8
-    # Only call set_num_threads if within NUMEXPR_MAX_THREADS limit to avoid warning
+    # C-Blosc2 uses BLOSC_NTHREADS, when set to a valid number, for every compression and
+    # decompression, whatever the nthreads in their parameters; start from it (as C-Blosc2 reads
+    # it), so that nthreads and the CParams/DParams defaults say how many threads are used
+    from .blosc2_ext import context_nthreads
+
+    nthreads = context_nthreads(nthreads)
+    # Only call set_num_threads if within NUMEXPR_MAX_THREADS limit to avoid warning, and not
+    # over the thread count that the user gave numexpr with NUMEXPR_NUM_THREADS
     numexpr_max_env = os.environ.get("NUMEXPR_MAX_THREADS")
     numexpr_max: int | None = None
     if numexpr_max_env is not None:
         with contextlib.suppress(ValueError):
             numexpr_max = int(numexpr_max_env)
-    if numexpr_max is None or nthreads <= numexpr_max:
-        numexpr.set_num_threads(nthreads)
+    if not os.environ.get("NUMEXPR_NUM_THREADS") and (numexpr_max is None or nthreads <= numexpr_max):
+        # BLOSC_NTHREADS can go beyond numexpr's own maximum, which it refuses with an error
+        numexpr.set_num_threads(min(nthreads, numexpr.MAX_THREADS))
 
 # This import must be before ndarray and schunk
 from .storage import (  # noqa: I001