From 9b64d4f859f83809471f58ab298b31fb8c16695c Mon Sep 17 00:00:00 2001 From: Igor Stadnyk Date: Sun, 16 Aug 2026 05:22:35 +0100 Subject: [PATCH 1/3] Fix empty Blosc buffer round-trips --- docs/release.md | 6 ++++++ src/numcodecs/blosc.pyx | 14 +++++++++++--- src/numcodecs/meson.build | 6 +++++- tests/test_blosc.py | 8 ++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/release.md b/docs/release.md index 78e8901c..2996de5a 100644 --- a/docs/release.md +++ b/docs/release.md @@ -14,6 +14,12 @@ ## Unreleased +### Fixes + +* Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip + while rejecting truncated or size-mismatched frames. System Blosc builds now require version + 1.16.0 or newer. By {user}`Igor Stadnyk `, {issue}`831`. + ### Maintenance * **Migrate build system from setuptools/setup.py to meson-python.** This replaces the diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index d61f7358..75b15be4 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -49,6 +49,7 @@ cdef extern from "blosc.h": int numinternalthreads) nogil int blosc_decompress_ctx(const void* src, void* dest, size_t destsize, int numinternalthreads) nogil + int blosc_cbuffer_validate(const void* cbuffer, size_t cbytes, size_t* nbytes) void blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes, size_t* blocksize) char* blosc_cbuffer_complib(const void* cbuffer) @@ -350,7 +351,7 @@ def decompress(source, dest=None): memoryview dest_mv Py_buffer* dest_pb char* dest_ptr - size_t nbytes, cbytes, blocksize + size_t nbytes # obtain source memoryview source_mv = ensure_contiguous_memoryview(source) @@ -359,8 +360,10 @@ def decompress(source, dest=None): # get source pointer source_ptr = source_pb.buf - # determine buffer size - blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) + # validate source and determine decompressed buffer size + ret = blosc_cbuffer_validate(source_ptr, source_pb.len, &nbytes) + if ret != 0: + raise RuntimeError('error during blosc decompression: %d' % ret) # setup destination buffer if dest is None: @@ -382,6 +385,11 @@ def decompress(source, dest=None): raise ValueError('destination buffer too small; expected at least %s, ' 'got %s' % (nbytes, dest_nbytes)) + # Blosc reports the number of decompressed bytes, so zero is successful + # for a validated empty frame rather than an error. + if nbytes == 0: + return dest + # perform decompression if _get_use_threads(): # allow blosc to use threads internally diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index eca2b2c6..117b67b6 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -153,7 +153,11 @@ zlib_dep = declare_dependency( ) # --- Vendored blosc --- -blosc_dep = dependency('blosc', required: get_option('system_blosc')) +blosc_dep = dependency( + 'blosc', + version: '>=1.16.0', + required: get_option('system_blosc'), +) if not blosc_dep.found() blosc_sources = files( diff --git a/tests/test_blosc.py b/tests/test_blosc.py index 537dc350..dc032119 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -75,6 +75,14 @@ def test_encode_decode(array, codec): check_encode_decode(array, codec) +def test_empty_encode_decode(use_threads): + blosc.use_threads = use_threads + try: + check_encode_decode(np.empty(0, dtype='u1'), Blosc()) + finally: + blosc.use_threads = None + + def test_config(): codec = Blosc(cname='zstd', clevel=3, shuffle=1) check_config(codec) From e3692a4ef95312030e888a140a363c17ccc94e05 Mon Sep 17 00:00:00 2001 From: Igor Stadnyk Date: Tue, 25 Aug 2026 14:49:51 +0100 Subject: [PATCH 2/3] Address Blosc decompression review feedback Preserve trailing-byte compatibility, keep empty-frame output failures loud, verify the full decompressed size, and expand regression and build-documentation coverage. Generated-by: OpenAI Codex --- docs/index.md | 2 ++ docs/release.md | 5 +++-- src/numcodecs/blosc.pyx | 29 +++++++++++++++++++---------- tests/test_blosc.py | 22 ++++++++++++++++++++++ 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/docs/index.md b/docs/index.md index bdaf1c8c..7170bdd5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,6 +52,8 @@ instead of the vendored copies:: --config-settings=setup-args=-Dsystem_zstd=enabled \ --config-settings=setup-args=-Dsystem_lz4=enabled +System Blosc builds require Blosc 1.16.0 or newer. + To work with Numcodecs source code in development, see the `contributing guide `_ for instructions on setting up a diff --git a/docs/release.md b/docs/release.md index 2996de5a..74f6c13b 100644 --- a/docs/release.md +++ b/docs/release.md @@ -17,8 +17,9 @@ ### Fixes * Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip - while rejecting truncated or size-mismatched frames. System Blosc builds now require version - 1.16.0 or newer. By {user}`Igor Stadnyk `, {issue}`831`. + while rejecting truncation and invalid frame headers and preserving support for trailing bytes. + System Blosc builds now require version 1.16.0 or newer. By + {user}`Igor Stadnyk `, {issue}`831`. ### Maintenance diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index 75b15be4..744cbbd0 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -19,6 +19,7 @@ from .abc import Codec cdef extern from "blosc.h": cdef enum: + BLOSC_MIN_HEADER_LENGTH, BLOSC_MAX_OVERHEAD, BLOSC_VERSION_STRING, BLOSC_VERSION_DATE, @@ -333,7 +334,8 @@ def decompress(source, dest=None): ---------- source : bytes-like Compressed data, including blosc header. Can be any object supporting the buffer - protocol. + protocol. Bytes after the first complete Blosc frame are ignored for backward + compatibility. dest : array-like, optional Object to decompress into. @@ -351,7 +353,7 @@ def decompress(source, dest=None): memoryview dest_mv Py_buffer* dest_pb char* dest_ptr - size_t nbytes + size_t nbytes, cbytes, blocksize # obtain source memoryview source_mv = ensure_contiguous_memoryview(source) @@ -360,10 +362,16 @@ def decompress(source, dest=None): # get source pointer source_ptr = source_pb.buf - # validate source and determine decompressed buffer size - ret = blosc_cbuffer_validate(source_ptr, source_pb.len, &nbytes) + # Read the declared frame size only after proving the complete header is present. + # Validate exactly that frame so trailing bytes remain backward compatible. + if source_pb.len < BLOSC_MIN_HEADER_LENGTH: + raise RuntimeError('invalid blosc frame: buffer is too small') + blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) + if cbytes > source_pb.len: + raise RuntimeError('invalid blosc frame: buffer is truncated') + ret = blosc_cbuffer_validate(source_ptr, cbytes, &nbytes) if ret != 0: - raise RuntimeError('error during blosc decompression: %d' % ret) + raise RuntimeError('invalid blosc frame: header validation failed') # setup destination buffer if dest is None: @@ -385,10 +393,11 @@ def decompress(source, dest=None): raise ValueError('destination buffer too small; expected at least %s, ' 'got %s' % (nbytes, dest_nbytes)) - # Blosc reports the number of decompressed bytes, so zero is successful - # for a validated empty frame rather than an error. - if nbytes == 0: - return dest + # Preserve the previous loud failure for a non-empty destination paired + # with an empty frame instead of returning an untouched output buffer. + if nbytes == 0 and dest_nbytes != 0: + raise RuntimeError('cannot decompress an empty blosc frame into a ' + 'non-empty destination buffer') # perform decompression if _get_use_threads(): @@ -403,7 +412,7 @@ def decompress(source, dest=None): pass # handle errors - if ret <= 0: + if ret < 0 or ret != nbytes: raise RuntimeError('error during blosc decompression: %d' % ret) return dest diff --git a/tests/test_blosc.py b/tests/test_blosc.py index dc032119..0464d570 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -42,6 +42,7 @@ # mix of shapes: 1D, 2D, 3D # mix of orders: C, F arrays = [ + np.empty(0, dtype='u1'), np.arange(1000, dtype='i4'), np.linspace(1000, 1001, 1000, dtype='f8'), np.random.normal(loc=1000, scale=1, size=(100, 10)), @@ -83,6 +84,27 @@ def test_empty_encode_decode(use_threads): blosc.use_threads = None +def test_empty_decode_rejects_nonempty_destination(): + codec = Blosc() + encoded = codec.encode(b'') + with pytest.raises(RuntimeError, match='non-empty destination buffer'): + codec.decode(encoded, out=bytearray(1)) + + +def test_decompress_allows_trailing_bytes(): + codec = Blosc() + original = b'some data to compress' + encoded = codec.encode(original) + assert codec.decode(encoded + b'padding') == original + + +def test_decompress_rejects_truncated_frame(): + codec = Blosc() + encoded = codec.encode(b'some data to compress') + with pytest.raises(RuntimeError, match='buffer is truncated'): + codec.decode(encoded[:-1]) + + def test_config(): codec = Blosc(cname='zstd', clevel=3, shuffle=1) check_config(codec) From 7f3a3d6279ff9a81759b7e72095df8ed3a335b01 Mon Sep 17 00:00:00 2001 From: Igor Stadnyk Date: Tue, 25 Aug 2026 14:53:47 +0100 Subject: [PATCH 3/3] Keep Blosc compatibility fixture ordering stable Append the empty-array case so existing fixture indices keep their previous meaning. Generated-by: OpenAI Codex --- tests/test_blosc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_blosc.py b/tests/test_blosc.py index 0464d570..ec4b5488 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -42,7 +42,6 @@ # mix of shapes: 1D, 2D, 3D # mix of orders: C, F arrays = [ - np.empty(0, dtype='u1'), np.arange(1000, dtype='i4'), np.linspace(1000, 1001, 1000, dtype='f8'), np.random.normal(loc=1000, scale=1, size=(100, 10)), @@ -56,6 +55,7 @@ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'), + np.empty(0, dtype='u1'), ]