diff --git a/docs/index.md b/docs/index.md index bdaf1c8c..7170bdd5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,6 +52,8 @@ instead of the vendored copies:: --config-settings=setup-args=-Dsystem_zstd=enabled \ --config-settings=setup-args=-Dsystem_lz4=enabled +System Blosc builds require Blosc 1.16.0 or newer. + To work with Numcodecs source code in development, see the `contributing guide `_ for instructions on setting up a diff --git a/docs/release.md b/docs/release.md index 78e8901c..74f6c13b 100644 --- a/docs/release.md +++ b/docs/release.md @@ -14,6 +14,13 @@ ## Unreleased +### Fixes + +* Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip + while rejecting truncation and invalid frame headers and preserving support for trailing bytes. + System Blosc builds now require version 1.16.0 or newer. By + {user}`Igor Stadnyk `, {issue}`831`. + ### Maintenance * **Migrate build system from setuptools/setup.py to meson-python.** This replaces the diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index d61f7358..744cbbd0 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -19,6 +19,7 @@ from .abc import Codec cdef extern from "blosc.h": cdef enum: + BLOSC_MIN_HEADER_LENGTH, BLOSC_MAX_OVERHEAD, BLOSC_VERSION_STRING, BLOSC_VERSION_DATE, @@ -49,6 +50,7 @@ cdef extern from "blosc.h": int numinternalthreads) nogil int blosc_decompress_ctx(const void* src, void* dest, size_t destsize, int numinternalthreads) nogil + int blosc_cbuffer_validate(const void* cbuffer, size_t cbytes, size_t* nbytes) void blosc_cbuffer_sizes(const void* cbuffer, size_t* nbytes, size_t* cbytes, size_t* blocksize) char* blosc_cbuffer_complib(const void* cbuffer) @@ -332,7 +334,8 @@ def decompress(source, dest=None): ---------- source : bytes-like Compressed data, including blosc header. Can be any object supporting the buffer - protocol. + protocol. Bytes after the first complete Blosc frame are ignored for backward + compatibility. dest : array-like, optional Object to decompress into. @@ -359,8 +362,16 @@ def decompress(source, dest=None): # get source pointer source_ptr = source_pb.buf - # determine buffer size + # Read the declared frame size only after proving the complete header is present. + # Validate exactly that frame so trailing bytes remain backward compatible. + if source_pb.len < BLOSC_MIN_HEADER_LENGTH: + raise RuntimeError('invalid blosc frame: buffer is too small') blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize) + if cbytes > source_pb.len: + raise RuntimeError('invalid blosc frame: buffer is truncated') + ret = blosc_cbuffer_validate(source_ptr, cbytes, &nbytes) + if ret != 0: + raise RuntimeError('invalid blosc frame: header validation failed') # setup destination buffer if dest is None: @@ -382,6 +393,12 @@ def decompress(source, dest=None): raise ValueError('destination buffer too small; expected at least %s, ' 'got %s' % (nbytes, dest_nbytes)) + # Preserve the previous loud failure for a non-empty destination paired + # with an empty frame instead of returning an untouched output buffer. + if nbytes == 0 and dest_nbytes != 0: + raise RuntimeError('cannot decompress an empty blosc frame into a ' + 'non-empty destination buffer') + # perform decompression if _get_use_threads(): # allow blosc to use threads internally @@ -395,7 +412,7 @@ def decompress(source, dest=None): pass # handle errors - if ret <= 0: + if ret < 0 or ret != nbytes: raise RuntimeError('error during blosc decompression: %d' % ret) return dest diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index eca2b2c6..117b67b6 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -153,7 +153,11 @@ zlib_dep = declare_dependency( ) # --- Vendored blosc --- -blosc_dep = dependency('blosc', required: get_option('system_blosc')) +blosc_dep = dependency( + 'blosc', + version: '>=1.16.0', + required: get_option('system_blosc'), +) if not blosc_dep.found() blosc_sources = files( diff --git a/tests/test_blosc.py b/tests/test_blosc.py index 537dc350..ec4b5488 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -55,6 +55,7 @@ np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[ns]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('M8[m]'), np.random.randint(-(2**63), -(2**63) + 20, size=1000, dtype='i8').view('m8[m]'), + np.empty(0, dtype='u1'), ] @@ -75,6 +76,35 @@ def test_encode_decode(array, codec): check_encode_decode(array, codec) +def test_empty_encode_decode(use_threads): + blosc.use_threads = use_threads + try: + check_encode_decode(np.empty(0, dtype='u1'), Blosc()) + finally: + blosc.use_threads = None + + +def test_empty_decode_rejects_nonempty_destination(): + codec = Blosc() + encoded = codec.encode(b'') + with pytest.raises(RuntimeError, match='non-empty destination buffer'): + codec.decode(encoded, out=bytearray(1)) + + +def test_decompress_allows_trailing_bytes(): + codec = Blosc() + original = b'some data to compress' + encoded = codec.encode(original) + assert codec.decode(encoded + b'padding') == original + + +def test_decompress_rejects_truncated_frame(): + codec = Blosc() + encoded = codec.encode(b'some data to compress') + with pytest.raises(RuntimeError, match='buffer is truncated'): + codec.decode(encoded[:-1]) + + def test_config(): codec = Blosc(cname='zstd', clevel=3, shuffle=1) check_config(codec)