From 0564675933eb20d1fb99ecd5c31de0d536e656fb Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Thu, 19 Nov 2026 17:15:37 -0600 Subject: [PATCH] Fail closed on decoder allocation and prefix bounds Reject unrepresentable compression bounds and handle allocation failure without exceptions. Decode size prefixes without alignment assumptions and reject truncated prefixes. Files changed: - lib/decoder/PayloadDecoder.cpp - tests/unittests/PayloadDecoderTests.cpp - tests/no-exceptions/no-exceptions-smoke.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e5009227-6b49-4803-8153-63d8d94037e3 --- lib/decoder/PayloadDecoder.cpp | 92 +++++--- tests/no-exceptions/no-exceptions-smoke.cpp | 52 ++++- tests/unittests/PayloadDecoderTests.cpp | 225 ++++++++++++++++++++ 3 files changed, 334 insertions(+), 35 deletions(-) diff --git a/lib/decoder/PayloadDecoder.cpp b/lib/decoder/PayloadDecoder.cpp index 25a0231ce..2c67dade0 100644 --- a/lib/decoder/PayloadDecoder.cpp +++ b/lib/decoder/PayloadDecoder.cpp @@ -33,6 +33,8 @@ MAT_NS_END #else #include #include +#include +#include #include #include #include @@ -391,7 +393,9 @@ namespace clienttelemetry { /// /// /// - bool Compress(const char* source, size_t sourceLen, char** dest, size_t& destLen, bool prependSize) + // Successful custom allocations must be compatible with delete[]. + bool Compress(const char* source, size_t sourceLen, char** dest, size_t& destLen, bool prependSize, + char* (*allocate)(size_t)) { if ((!source) || (!sourceLen)) { @@ -401,38 +405,48 @@ namespace clienttelemetry { *dest = NULL; destLen = 0; - // Compressing variables - uLong compSize = compressBound((uLong)sourceLen); + const auto inputSize = static_cast(sourceLen); + if (inputSize > static_cast(std::numeric_limits::max()) || + (prependSize && inputSize > static_cast(std::numeric_limits::max()))) + { + return false; + } - // Allocate memory for the new compressed buffer - size_t reserved = ((unsigned)prependSize * sizeof(uint32_t)); - char* compBody = new char[std::max(compSize, ((uLong)sourceLen)) + reserved]; - if (compBody != NULL) + uLong compSize = compressBound(static_cast(sourceLen)); + const size_t reserved = prependSize ? sizeof(uint32_t) : 0; + const size_t maxAllocation = static_cast(std::numeric_limits::max()); + if (compSize == std::numeric_limits::max() || compSize < sourceLen || + sourceLen > maxAllocation - reserved || compSize > maxAllocation - reserved) { - if (prependSize) - { - // Remember source uncompressed size if requested - uint32_t *s = (uint32_t*)(compBody); - (*s) = (uint32_t)sourceLen; // truncate this to 32-bit, we do not support 3+ TB blobs - } - // Deflate - int res = compress2((Bytef *)(compBody + reserved), &compSize, (Bytef *)source, (uLong)sourceLen, Z_BEST_SPEED); - if (res != Z_OK) - { - TEST_LOG_ERROR("Compression failed, error=%u", res); - delete[] compBody; - compBody = NULL; - return false; - } - else - { - *dest = compBody; - destLen = compSize + reserved; - return true; - } + return false; } - // OOM - return false; + + char* compBody = allocate(std::max(static_cast(compSize), sourceLen) + reserved); + if (compBody == nullptr) + { + return false; + } + if (prependSize) + { + const uint32_t originalSize = static_cast(sourceLen); + std::memcpy(compBody, &originalSize, sizeof(originalSize)); + } + int res = compress2((Bytef *)(compBody + reserved), &compSize, (Bytef *)source, (uLong)sourceLen, Z_BEST_SPEED); + if (res != Z_OK) + { + TEST_LOG_ERROR("Compression failed, error=%u", res); + delete[] compBody; + return false; + } + *dest = compBody; + destLen = compSize + reserved; + return true; + } + + bool Compress(const char* source, size_t sourceLen, char** dest, size_t& destLen, bool prependSize) + { + return Compress(source, sourceLen, dest, destLen, prependSize, + [](size_t size) { return new (std::nothrow) char[size]; }); } /// @@ -460,13 +474,23 @@ namespace clienttelemetry { // Get uncompressed size at zero offset. if (sizeAtZeroIndex) { - uint32_t s32 = *((uint32_t*)(source)); - uint64_t s64 = (sourceLen >= sizeof(uint64_t)) ? *((uint64_t*)(source)) : 0; + if (sourceLen < sizeof(uint32_t)) + { + destLen = 0; + return false; + } + uint32_t s32 = 0; + std::memcpy(&s32, source, sizeof(s32)); // If we are reading 64-bit generated legacy DB, step 32-bit forward to // skip zero-padding in most-significant DWORD on Intel architecture - if ((s64 - s32) == 0) + if (sourceLen >= sizeof(uint64_t)) { - reserved += sizeof(uint32_t); + uint64_t s64 = 0; + std::memcpy(&s64, source, sizeof(s64)); + if (s64 == s32) + { + reserved += sizeof(uint32_t); + } } destLen = s32; } diff --git a/tests/no-exceptions/no-exceptions-smoke.cpp b/tests/no-exceptions/no-exceptions-smoke.cpp index d85307af3..79773285f 100644 --- a/tests/no-exceptions/no-exceptions-smoke.cpp +++ b/tests/no-exceptions/no-exceptions-smoke.cpp @@ -8,9 +8,11 @@ #include "http/HttpResponseDecoder.hpp" #include "utils/ZlibUtils.hpp" +#include #include #include #include +#include #if HAVE_EXCEPTIONS #error This regression executable must be compiled without C++ exceptions. @@ -19,9 +21,25 @@ #error This regression executable requires JSON and zlib support. #endif +namespace +{ + std::atomic failNextNothrowArrayAllocation {false}; +} + +void* operator new[](std::size_t size, const std::nothrow_t&) noexcept +{ + if (failNextNothrowArrayAllocation.exchange(false)) + { + return nullptr; + } + return ::operator new[](size); +} + namespace clienttelemetry { namespace data { namespace v3 { + bool Compress(const char*, size_t, char**, size_t&, bool); + bool Compress(const char*, size_t, char**, size_t&, bool, char* (*)(size_t)); bool Expand(const char*, size_t, char**, size_t&, bool); bool Expand(const char*, size_t, char**, size_t&, bool, char* (*)(size_t)); } @@ -151,6 +169,14 @@ int main() "oversized expansion must fail"); check(output == nullptr && outputSize == 0, "failed expansion must clear outputs"); char previousOutput = 0; + for (size_t length = 1; length < sizeof(uint32_t); ++length) + { + output = &previousOutput; + outputSize = 32; + check(!clienttelemetry::data::v3::Expand(invalidCompressed, length, &output, outputSize, true), + "short size prefix must fail"); + check(output == nullptr && outputSize == 0, "short size prefix must clear outputs"); + } output = &previousOutput; outputSize = 32; static size_t requestedSize = 0; @@ -161,6 +187,30 @@ int main() }), "representable allocation failure must return false"); check(requestedSize == 32, "failure injection must reach the allocator"); check(output == nullptr && outputSize == 0, "allocation failure must clear an existing output and its length"); - std::puts("Passed no-exceptions configuration, response, profile, numeric-limit, and expansion smoke checks."); + output = &previousOutput; + outputSize = 32; + requestedSize = 0; + check(!clienttelemetry::data::v3::Compress(invalidCompressed, sizeof(invalidCompressed), &output, outputSize, false, + [](size_t size) -> char* { + requestedSize = size; + return nullptr; + }), "compression allocation failure must return false"); + check(requestedSize > sizeof(invalidCompressed), "compression failure injection must reach the allocator"); + check(output == nullptr && outputSize == 0, "failed compression must clear output and its length"); + output = &previousOutput; + outputSize = 32; + failNextNothrowArrayAllocation = true; + const bool compressed = clienttelemetry::data::v3::Compress( + invalidCompressed, sizeof(invalidCompressed), &output, outputSize, false); + check(!failNextNothrowArrayAllocation, "default compression must use the nothrow array allocator"); + check(!compressed && output == nullptr && outputSize == 0, + "default compression must report allocation failure without exceptions"); + output = &previousOutput; + outputSize = 32; + check(!clienttelemetry::data::v3::Compress(invalidCompressed, std::numeric_limits::max(), + &output, outputSize, true), + "unrepresentable compression size must fail"); + check(output == nullptr && outputSize == 0, "oversized compression must clear output and its length"); + std::puts("Passed no-exceptions configuration, response, profile, numeric-limit, and compression smoke checks."); return EXIT_SUCCESS; } diff --git a/tests/unittests/PayloadDecoderTests.cpp b/tests/unittests/PayloadDecoderTests.cpp index 6a3d8f6a2..5be428b5f 100644 --- a/tests/unittests/PayloadDecoderTests.cpp +++ b/tests/unittests/PayloadDecoderTests.cpp @@ -5,16 +5,23 @@ #include "common/Common.hpp" #include "PayloadDecoder.hpp" #include "utils/ZlibUtils.hpp" +#include +#include #include +#include #if defined(HAVE_MAT_ZLIB) && defined(HAVE_MAT_JSONHPP) #include "bond/All.hpp" #include "bond/generated/CsProtocol_writers.hpp" #include +#include namespace clienttelemetry { namespace data { namespace v3 { + bool Compress(const char* source, size_t sourceLen, char** dest, size_t& destLen, bool prependSize); + bool Compress(const char* source, size_t sourceLen, char** dest, size_t& destLen, bool prependSize, + char* (*allocate)(size_t)); bool Expand(const char* source, size_t sourceLen, char** dest, size_t& destLen, bool sizeAtZeroIndex); bool Expand(const char* source, size_t sourceLen, char** dest, size_t& destLen, bool sizeAtZeroIndex, char* (*allocate)(size_t)); @@ -147,6 +154,224 @@ TEST(PayloadDecoderTests, DecodeRecord_ValidUtf8_IsPreserved) } #if defined(HAVE_MAT_ZLIB) && defined(HAVE_MAT_JSONHPP) +TEST(PayloadDecoderTests, Compress_InvalidInputLeavesOutputUnchanged) +{ + const char source[] = "sample"; + char previousOutput = 0; + char* output = &previousOutput; + size_t outputSize = 32; + EXPECT_FALSE(clienttelemetry::data::v3::Compress(nullptr, sizeof(source), &output, outputSize, false)); + EXPECT_FALSE(clienttelemetry::data::v3::Compress(source, 0, &output, outputSize, false)); + EXPECT_EQ(output, &previousOutput); + EXPECT_EQ(outputSize, 32u); +} + +TEST(PayloadDecoderTests, Compress_AllocationFailureClearsExistingOutput) +{ + const char source[] = "sample"; + for (bool prependSize : {false, true}) + { + char previousOutput = 0; + char* output = &previousOutput; + size_t outputSize = 32; + static size_t requestedSize = 0; + requestedSize = 0; + EXPECT_FALSE(clienttelemetry::data::v3::Compress(source, sizeof(source), &output, outputSize, prependSize, + [](size_t size) -> char* { + requestedSize = size; + return nullptr; + })); + EXPECT_GT(requestedSize, sizeof(source)); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(outputSize, 0u); + } +} + +TEST(PayloadDecoderTests, Compress_RejectsUnrepresentableSizeBeforeAllocation) +{ + const char source[] = "sample"; + for (bool prependSize : {false, true}) + { + char previousOutput = 0; + char* output = &previousOutput; + size_t outputSize = 32; + static bool allocated = false; + allocated = false; + EXPECT_FALSE(clienttelemetry::data::v3::Compress(source, std::numeric_limits::max(), + &output, outputSize, prependSize, [](size_t) -> char* { + allocated = true; + return nullptr; + })); + EXPECT_FALSE(allocated); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(outputSize, 0u); + } +} + +TEST(PayloadDecoderTests, Compress_RejectsSaturatedZlibBound) +{ + const auto maxZlibSize = static_cast(std::numeric_limits::max()); + if (maxZlibSize <= std::numeric_limits::max()) + { + const char source[] = "sample"; + char* output = nullptr; + size_t outputSize = 0; + static bool allocated = false; + allocated = false; + EXPECT_FALSE(clienttelemetry::data::v3::Compress(source, static_cast(maxZlibSize), + &output, outputSize, false, [](size_t) -> char* { + allocated = true; + return nullptr; + })); + EXPECT_FALSE(allocated); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(outputSize, 0u); + } +} + +TEST(PayloadDecoderTests, Compress_RejectsUnrepresentableAllocationBound) +{ + const auto maxAllocation = static_cast(std::numeric_limits::max()); + if (maxAllocation <= static_cast(std::numeric_limits::max())) + { + const char source[] = "sample"; + char* output = nullptr; + size_t outputSize = 0; + static bool allocated = false; + allocated = false; + EXPECT_FALSE(clienttelemetry::data::v3::Compress(source, static_cast(maxAllocation), + &output, outputSize, false, [](size_t) -> char* { + allocated = true; + return nullptr; + })); + EXPECT_FALSE(allocated); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(outputSize, 0u); + } +} + +TEST(PayloadDecoderTests, Compress_RejectsSizeThatCannotFitPrefix) +{ + const auto oversized = static_cast(std::numeric_limits::max()) + 1; + if (oversized <= std::numeric_limits::max()) + { + const char source[] = "sample"; + char* output = nullptr; + size_t outputSize = 0; + static bool allocated = false; + allocated = false; + EXPECT_FALSE(clienttelemetry::data::v3::Compress(source, static_cast(oversized), + &output, outputSize, true, [](size_t) -> char* { + allocated = true; + return nullptr; + })); + EXPECT_FALSE(allocated); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(outputSize, 0u); + + if (oversized <= static_cast(std::numeric_limits::max()) && + oversized < static_cast(std::numeric_limits::max())) + { + EXPECT_FALSE(clienttelemetry::data::v3::Compress(source, static_cast(oversized), + &output, outputSize, false, [](size_t) -> char* { + allocated = true; + return nullptr; + })); + EXPECT_TRUE(allocated); + } + } +} + +TEST(PayloadDecoderTests, Compress_PreservesZlibOutputWithAndWithoutSizePrefix) +{ + const char source[] = "sample compression payload"; + std::vector expected(compressBound(sizeof(source))); + uLongf expectedSize = static_cast(expected.size()); + ASSERT_EQ(compress2(expected.data(), &expectedSize, reinterpret_cast(source), + sizeof(source), Z_BEST_SPEED), Z_OK); + + for (bool prependSize : {false, true}) + { + char* output = nullptr; + size_t outputSize = 0; + ASSERT_TRUE(clienttelemetry::data::v3::Compress(source, sizeof(source), &output, outputSize, prependSize)); + std::unique_ptr owned(output); + const size_t reserved = prependSize ? sizeof(uint32_t) : 0; + ASSERT_EQ(outputSize, expectedSize + reserved); + EXPECT_EQ(std::memcmp(output + reserved, expected.data(), expectedSize), 0); + if (prependSize) + { + uint32_t originalSize = 0; + std::memcpy(&originalSize, output, sizeof(originalSize)); + EXPECT_EQ(originalSize, sizeof(source)); + + std::vector unaligned(outputSize + 1); + std::memcpy(unaligned.data() + 1, output, outputSize); + char* expanded = nullptr; + size_t expandedSize = 0; + ASSERT_TRUE(clienttelemetry::data::v3::Expand( + unaligned.data() + 1, outputSize, &expanded, expandedSize, true)); + std::unique_ptr expandedOwner(expanded); + ASSERT_EQ(expandedSize, sizeof(source)); + EXPECT_EQ(std::memcmp(expanded, source, sizeof(source)), 0); + + const uint16_t endianMarker = 1; + if (*reinterpret_cast(&endianMarker) == 1) + { + std::vector legacy(outputSize + sizeof(uint32_t) + 1, 0); + std::memcpy(legacy.data() + 1, output, sizeof(uint32_t)); + std::memcpy(legacy.data() + 1 + sizeof(uint64_t), output + sizeof(uint32_t), + outputSize - sizeof(uint32_t)); + expanded = nullptr; + expandedSize = 0; + ASSERT_TRUE(clienttelemetry::data::v3::Expand(legacy.data() + 1, + outputSize + sizeof(uint32_t), &expanded, expandedSize, true)); + std::unique_ptr legacyOwner(expanded); + ASSERT_EQ(expandedSize, sizeof(source)); + EXPECT_EQ(std::memcmp(expanded, source, sizeof(source)), 0); + } + } + char restored[sizeof(source)] {}; + uLongf restoredSize = sizeof(restored); + ASSERT_EQ(uncompress(reinterpret_cast(restored), &restoredSize, + reinterpret_cast(output + reserved), + static_cast(outputSize - reserved)), Z_OK); + ASSERT_EQ(restoredSize, sizeof(source)); + EXPECT_EQ(std::memcmp(restored, source, sizeof(source)), 0); + } +} + +TEST(PayloadDecoderTests, Expand_RejectsShortOrIncompleteSizePrefixes) +{ + char previousOutput = 0; + char* output = &previousOutput; + size_t outputSize = 32; + const char source = 0; + EXPECT_FALSE(clienttelemetry::data::v3::Expand(&source, 0, &output, outputSize, true)); + EXPECT_EQ(output, &previousOutput); + EXPECT_EQ(outputSize, 32u); + + for (size_t length = 1; length <= sizeof(uint64_t); ++length) + { + std::vector prefix(length, 0); + output = &previousOutput; + outputSize = 32; + EXPECT_FALSE(clienttelemetry::data::v3::Expand(prefix.data(), prefix.size(), &output, outputSize, true)); + EXPECT_EQ(output, nullptr); + EXPECT_EQ(outputSize, 0u); + + if (length >= sizeof(uint32_t)) + { + const uint32_t declaredSize = 1; + std::memcpy(prefix.data(), &declaredSize, sizeof(declaredSize)); + output = &previousOutput; + outputSize = 32; + EXPECT_FALSE(clienttelemetry::data::v3::Expand(prefix.data(), prefix.size(), &output, outputSize, true)); + EXPECT_EQ(output, nullptr); + } + } +} + TEST(PayloadDecoderTests, Expand_UnrepresentableSizeClearsOutput) { const char source[] = "invalid";