From f2215b7df3e223a8bd00965d26a419dd7fd77995 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Thu, 5 Nov 2026 22:46:19 +0100 Subject: [PATCH] Refuse buffers of another typesize in Python pre/postfilters general_postfilter, general_prefilter and general_filler hand Python arrays of size // typesize items of the callback's dtype. When the typesize of the buffer is smaller than the dtype's itemsize, the arrays reach past the end of the buffer and the callback writes there: C-Blosc2 before Blosc/c-blosc2#857 decompresses vlmetalayers with the super-chunk's postfilter, so reading schunk.vlmeta[...] of an SChunk with an int32 postfilter writes 36 bytes into a 9-byte msgpack buffer, and glibc aborts later ("corrupted size vs. prev_size"). The same happens with an SChunk whose typesize is smaller than the callback dtype. A larger typesize gives arrays that cover part of the buffer only, so part of the result is silently left unwritten (or zero for prefilters). Store the itemsize with the callback and raise ValueError ("postfilter: data typesize 1 does not match dtype itemsize 4"), before building the arrays or writing anything, when the typesize of the buffer differs. C-Blosc2 then stops, and the callers raise the pending exception after the C call (_raise_callback_error(), #766). SChunk.filler() now removes the filler also when an append fails: it stayed set before, with only the id of its inputs tuple, which a later append would have used after the tuple was gone. The tests use callbacks that only record their calls, so they cannot overrun anything even without the guard, and run with the GIL kept and released: postfilter, prefilter and filler with typesize 1 and 8 for int32, and the vlmeta read/to_cframe() of an SChunk with a postfilter (with the bundled C-Blosc2 the guard refuses the vlmetalayer; with Blosc/c-blosc2#857 the content is read without the postfilter). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DiPFztdpnKMJkzrjT36BuP --- src/blosc2/blosc2_ext.pyx | 21 +++++++++++ src/blosc2/schunk.py | 30 +++++++++------ tests/test_postfilters.py | 79 +++++++++++++++++++++++++++++++++++++++ tests/test_prefilters.py | 58 ++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 12 deletions(-) diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx index de90e254c..7d51d2990 100644 --- a/src/blosc2/blosc2_ext.pyx +++ b/src/blosc2/blosc2_ext.pyx @@ -825,12 +825,14 @@ ctypedef struct user_filters_udata: int input_cdtype int output_cdtype int32_t chunkshape + int32_t itemsize # of both dtypes ctypedef struct filler_udata: char* py_func uintptr_t inputs_id int output_cdtype int32_t chunkshape + int32_t itemsize # of the output dtype ctypedef struct udf_udata: char* py_func @@ -4793,6 +4795,7 @@ cdef class SChunk: postf_udata.input_cdtype = dtype_input.num postf_udata.output_cdtype = dtype_output.num postf_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize + postf_udata.itemsize = dtype_input.itemsize postparams.user_data = postf_udata dparams.postparams = postparams @@ -4847,6 +4850,7 @@ cdef class SChunk: fill_udata.inputs_id = inputs_id fill_udata.output_cdtype = np.dtype(dtype_output).num fill_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize + fill_udata.itemsize = np.dtype(dtype_output).itemsize preparams.user_data = fill_udata cparams.preparams = preparams @@ -4881,6 +4885,7 @@ cdef class SChunk: pref_udata.input_cdtype = dtype_input.num pref_udata.output_cdtype = dtype_output.num pref_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize + pref_udata.itemsize = dtype_input.itemsize preparams.user_data = pref_udata cparams.preparams = preparams @@ -5002,9 +5007,18 @@ cdef class SChunk: blosc2_schunk_free(schunk_ptr) +# The arrays that general_postfilter, general_filler and general_prefilter hand to Python have +# size // typesize items of the callback's dtype, so they only fit buffers whose typesize is the +# dtype's itemsize. Other buffers (C-Blosc2 before Blosc/c-blosc2#857 passes vlmetalayer content +# to the postfilter, and a chunk can have another typesize than its SChunk) get a ValueError, raised +# before anything is written and re-raised by the caller after the C call (_raise_callback_error()). + # postfilter cdef int general_postfilter(blosc2_postfilter_params *params) with gil: cdef user_filters_udata *udata = params.user_data + if params.typesize != udata.itemsize: + raise ValueError(f"postfilter: data typesize {params.typesize} does not match " + f"dtype itemsize {udata.itemsize}") cdef int nd = 1 cdef np.npy_intp dims = params.size // params.typesize input = np.PyArray_SimpleNewFromData(nd, &dims, udata.input_cdtype, params.input) @@ -5018,6 +5032,9 @@ cdef int general_postfilter(blosc2_postfilter_params *params) with gil: # filler cdef int general_filler(blosc2_prefilter_params *params): cdef filler_udata *udata = params.user_data + if params.output_typesize != udata.itemsize: + raise ValueError(f"filler: data typesize {params.output_typesize} does not match " + f"dtype itemsize {udata.itemsize}") cdef int nd = 1 cdef np.npy_intp dims = params.output_size // params.output_typesize @@ -5666,6 +5683,10 @@ def nelem_from_inputs(inputs_tuple, nelem=None): # prefilter cdef int general_prefilter(blosc2_prefilter_params *params): cdef user_filters_udata *udata = params.user_data + # Typesize check: see general_postfilter + if params.output_typesize != udata.itemsize: + raise ValueError(f"prefilter: data typesize {params.output_typesize} does not match " + f"dtype itemsize {udata.itemsize}") cdef int nd = 1 cdef np.npy_intp dims = params.output_size // params.output_typesize diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 33452acae..ef3c0d945 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -1538,7 +1538,8 @@ def postfilter(self, input_dtype: np.dtype, output_dtype: np.dtype = None) -> No * `nthreads` must be 1 when decompressing. * The :paramref:`input_dtype` itemsize must be the same as the - :paramref:`output_dtype` itemsize. + :paramref:`output_dtype` itemsize, and the typesize of the chunks: + decompressing a chunk with another typesize fails. See Also -------- @@ -1640,6 +1641,7 @@ def filler(self, inputs_tuple: tuple[tuple], schunk_dtype: np.dtype, nelem: int Notes ----- * Compression `nthreads` must be 1 when using this. + * The :paramref:`schunk_dtype` itemsize must be the typesize of the SChunk. * This does not need to be removed from the created `SChunk` instance. See Also @@ -1674,16 +1676,19 @@ def initialize(func): raise ValueError("Cannot apply a filler to a non empty SChunk") nelem_ = blosc2_ext.nelem_from_inputs(inputs_tuple, nelem) super(SChunk, self)._set_filler(func, id(inputs_tuple), schunk_dtype) - chunksize = self.chunksize - written_nbytes = 0 - nbytes = nelem_ * self.typesize - while written_nbytes < nbytes: - chunk = np.zeros(chunksize // self.typesize, dtype=schunk_dtype) - self.append_data(chunk) - written_nbytes += chunksize - if (nbytes - written_nbytes) < self.chunksize: - chunksize = nbytes - written_nbytes - self.remove_prefilter(func.__name__) + try: + chunksize = self.chunksize + written_nbytes = 0 + nbytes = nelem_ * self.typesize + while written_nbytes < nbytes: + chunk = np.zeros(chunksize // self.typesize, dtype=schunk_dtype) + self.append_data(chunk) + written_nbytes += chunksize + if (nbytes - written_nbytes) < self.chunksize: + chunksize = nbytes - written_nbytes + finally: + # Also after a failed append: the filler only keeps the id of inputs_tuple + self.remove_prefilter(func.__name__) def exec_func(*args): func(*args) @@ -1719,7 +1724,8 @@ def prefilter(self, input_dtype: np.dtype, output_dtype: np.dtype = None) -> Non * `nthreads` must be 1 when compressing. * The :paramref:`input_dtype` itemsize must be the same as the - :paramref:`output_dtype` itemsize. + :paramref:`output_dtype` itemsize, and the typesize of the SChunk: + compressing with another typesize fails. See Also -------- diff --git a/tests/test_postfilters.py b/tests/test_postfilters.py index 6ed66bbc2..677861cb0 100644 --- a/tests/test_postfilters.py +++ b/tests/test_postfilters.py @@ -86,3 +86,82 @@ def postf3(input, output, offset): assert np.array_equal(data, res) blosc2.remove_urlpath(urlpath) + + +# The postfilters below only record their calls and never touch their arrays, so a call with +# arrays that do not fit the buffers cannot overrun them, with or without the typesize guard. +# Each test runs with the GIL kept and released around the C calls. + + +@pytest.fixture(params=[False, True], ids=["gil-held", "gil-released"]) +def releasegil(request): + old = blosc2.set_releasegil(request.param) + yield + blosc2.set_releasegil(old) + + +@pytest.mark.parametrize("typesize", [1, 8]) +def test_postfilter_wrong_typesize(releasegil, typesize): + # int32 arrays of size // typesize items would overrun a typesize 1 buffer, and cover only + # half of a typesize 8 one: the postfilter refuses such chunks + data = np.arange(1000, dtype=np.int32) + schunk = blosc2.SChunk( + chunksize=data.nbytes, + data=data, + cparams={"typesize": typesize, "nthreads": 1}, + dparams={"nthreads": 1}, + ) + calls = [] + + @schunk.postfilter(np.int32) + def postf_wrong_typesize(input, output, offset): + calls.append(input.size) + + with pytest.raises( + ValueError, match=f"postfilter: data typesize {typesize} does not match dtype itemsize 4" + ): + schunk.decompress_chunk(0) + assert calls == [] + + schunk.remove_postfilter("postf_wrong_typesize") + assert np.array_equal(np.frombuffer(schunk.decompress_chunk(0), dtype=np.int32), data) + + +def test_postfilter_vlmeta(releasegil): + # The postfilter is for chunk data, not for the msgpack content (typesize 1) of a vlmetalayer + data = np.arange(2000, dtype=np.int32) + schunk = blosc2.SChunk( + chunksize=4000, + data=data, + contiguous=False, + cparams={"typesize": 4, "nthreads": 1}, + dparams={"nthreads": 1}, + ) + schunk.vlmeta["info"] = {"answer": 42} + calls = [] + + @schunk.postfilter(np.int32) + def postf_vlmeta(input, output, offset): + calls.append(input.size) + + # C-Blosc2 before Blosc/c-blosc2#857 decompresses vlmetalayers with the postfilter, which then + # refuses them; with it the content is read without the postfilter. Either way the callback + # never sees the content. + refused = "postfilter: data typesize 1 does not match dtype itemsize 4" + try: + info = schunk.vlmeta["info"] + except ValueError as e: + info = str(e) + assert info in ({"answer": 42}, refused) + assert calls == [] + + # Serializing an in-memory sparse super-chunk copies its vlmetalayers the same way + try: + copy = blosc2.schunk_from_cframe(schunk.to_cframe(), copy=True) + info = copy.vlmeta["info"] + except ValueError as e: + info = str(e) + assert info in ({"answer": 42}, refused) + + schunk.remove_postfilter("postf_vlmeta") + assert schunk.vlmeta["info"] == {"answer": 42} diff --git a/tests/test_prefilters.py b/tests/test_prefilters.py index d2da4365f..3fb77274a 100644 --- a/tests/test_prefilters.py +++ b/tests/test_prefilters.py @@ -207,3 +207,61 @@ def pref3(input, output, offset): assert np.array_equal(new_data, res) blosc2.remove_urlpath(urlpath) + + +# The prefilters below only record their calls and never touch their arrays, so a call with +# arrays that do not fit the buffers cannot overrun them, with or without the typesize guard. +# Each test runs with the GIL kept and released around the C calls. + + +@pytest.fixture(params=[False, True], ids=["gil-held", "gil-released"]) +def releasegil(request): + old = blosc2.set_releasegil(request.param) + yield + blosc2.set_releasegil(old) + + +@pytest.mark.parametrize("typesize", [1, 8]) +def test_prefilter_wrong_typesize(releasegil, typesize): + # int32 arrays of size // typesize items would overrun a typesize 1 buffer, and cover only + # half of a typesize 8 one: the prefilter refuses such chunks + data = np.arange(1000, dtype=np.int32) + schunk = blosc2.SChunk(chunksize=data.nbytes, cparams={"typesize": typesize, "nthreads": 1}) + calls = [] + + @schunk.prefilter(np.int32) + def pref_wrong_typesize(input, output, offset): + calls.append(input.size) + + with pytest.raises( + ValueError, match=f"prefilter: data typesize {typesize} does not match dtype itemsize 4" + ): + schunk.append_data(data) + assert calls == [] + assert schunk.nchunks == 0 + + schunk.remove_prefilter("pref_wrong_typesize") + schunk.append_data(data) + assert np.array_equal(np.frombuffer(schunk.decompress_chunk(0), dtype=np.int32), data) + + +@pytest.mark.parametrize("typesize", [1, 8]) +def test_filler_wrong_typesize(releasegil, typesize): + # Same for a filler, which then must not stay set: it keeps only the id of its inputs + data = np.arange(1000, dtype=np.int32) + schunk = blosc2.SChunk(chunksize=data.nbytes, cparams={"typesize": typesize, "nthreads": 1}) + calls = [] + + with pytest.raises( + ValueError, match=f"filler: data typesize {typesize} does not match dtype itemsize 4" + ): + + @schunk.filler(((data, np.int32),), np.int32) + def fill_wrong_typesize(inputs_tuple, output, offset): + calls.append(output.size) + + assert calls == [] + assert schunk.nchunks == 0 + assert "fill_wrong_typesize" not in blosc2.prefilter_funcs + schunk.append_data(data) + assert schunk.nchunks == 1