From 17cbdd7ca70e8626e06df09872529d984dc987ff Mon Sep 17 00:00:00 2001 From: kid <19265318+u70b3@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:58:00 +0000 Subject: [PATCH 1/2] Fix ARM char portability --- src/paimon/common/data/binary_row_test.cpp | 8 +- src/paimon/common/data/binary_string.cpp | 10 +- src/paimon/common/data/binary_string.h | 2 +- .../bitmap/bitmap_file_index_meta_test.cpp | 53 ++-- .../bitmap/bitmap_file_index_test.cpp | 228 ++++++++++-------- ...bit_slice_index_bitmap_file_index_test.cpp | 175 +++++++------- .../file_index/file_index_format_test.cpp | 13 +- .../io/data_input_output_stream_test.cpp | 28 +-- .../binary_row_partition_computer_test.cpp | 6 +- .../utils/data_converter_utils_test.cpp | 5 +- src/paimon/common/utils/murmurhash_utils.h | 8 +- 11 files changed, 284 insertions(+), 252 deletions(-) diff --git a/src/paimon/common/data/binary_row_test.cpp b/src/paimon/common/data/binary_row_test.cpp index 17ee3331..e04c18dc 100644 --- a/src/paimon/common/data/binary_row_test.cpp +++ b/src/paimon/common/data/binary_row_test.cpp @@ -341,10 +341,10 @@ TEST_F(BinaryRowTest, TestBinary) { auto pool = GetDefaultPool(); BinaryRow row(2); BinaryRowWriter writer(&row, 0, pool.get()); - char chars1[3] = {1, -1, 5}; - char chars2[8] = {1, -1, 5, 5, 1, 5, 1, 5}; - std::string str1(chars1, 3); - std::string str2(chars2, 8); + std::uint8_t chars1[3] = {1, 0xff, 5}; + std::uint8_t chars2[8] = {1, 0xff, 5, 5, 1, 5, 1, 5}; + std::string str1(reinterpret_cast(chars1), 3); + std::string str2(reinterpret_cast(chars2), 8); Bytes bytes1(str1, pool.get()); Bytes bytes2(str2, pool.get()); diff --git a/src/paimon/common/data/binary_string.cpp b/src/paimon/common/data/binary_string.cpp index 5dc56b3e..51457e98 100644 --- a/src/paimon/common/data/binary_string.cpp +++ b/src/paimon/common/data/binary_string.cpp @@ -74,17 +74,17 @@ std::string BinaryString::ToString() const { return ret; } -int32_t BinaryString::NumBytesForFirstByte(char b) { - if (b >= 0) { +int32_t BinaryString::NumBytesForFirstByte(std::uint8_t b) { + if ((b & 0x80) == 0) { // 1 byte, 7 bits: 0xxxxxxx return 1; - } else if ((b >> 5) == -2 && (b & 0x1e) != 0) { + } else if ((b & 0xe0) == 0xc0 && (b & 0x1e) != 0) { // 2 bytes, 11 bits: 110xxxxx 10xxxxxx return 2; - } else if ((b >> 4) == -2) { + } else if ((b & 0xf0) == 0xe0) { // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx return 3; - } else if ((b >> 3) == -2) { + } else if ((b & 0xf8) == 0xf0) { // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx return 4; } else { diff --git a/src/paimon/common/data/binary_string.h b/src/paimon/common/data/binary_string.h index 2de9b588..57c80f83 100644 --- a/src/paimon/common/data/binary_string.h +++ b/src/paimon/common/data/binary_string.h @@ -138,7 +138,7 @@ class PAIMON_EXPORT BinaryString : public BinarySection { /// @return the number of bytes for a code point with the first byte as b. /// @param b The first byte of a code point - static int32_t NumBytesForFirstByte(char b); + static int32_t NumBytesForFirstByte(std::uint8_t b); private: char GetByteOneSegment(int32_t i) const; diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_meta_test.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index_meta_test.cpp index 66fe4b87..3381d9e2 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index_meta_test.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_meta_test.cpp @@ -18,6 +18,7 @@ #include "paimon/common/file_index/bitmap/bitmap_file_index_meta.h" +#include #include #include #include @@ -67,13 +68,14 @@ TEST(BitmapFileIndexMetaTest, TestStringType) { }; { // test v1 version - std::vector index_bytes = { - 1, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, 97, 0, 0, 0, 20, 0, - 0, 0, 1, 98, -1, -1, -1, -3, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, - 1, 0, 3, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; - - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 1, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, + 97, 0, 0, 0, 20, 0, 0, 0, 1, 98, 0xff, 0xff, 0xff, 0xfd, 58, 48, 0, 0, + 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, + 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; + + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); // skip version ASSERT_OK(input_stream->Seek(1, SeekOrigin::FS_SEEK_SET)); BitmapFileIndexMetaV1 index_meta(FieldType::STRING, 0, index_bytes.size(), @@ -83,15 +85,16 @@ TEST(BitmapFileIndexMetaTest, TestStringType) { } { // test v2 version - std::vector index_bytes = { - 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, - 1, 0, 0, 0, 1, 97, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 2, 0, 0, 0, - 1, 97, 0, 0, 0, 20, 0, 0, 0, 20, 0, 0, 0, 1, 98, -1, -1, -1, -3, -1, -1, - -1, -1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, - 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; - - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 20, + 0, 0, 0, 1, 0, 0, 0, 1, 97, 0, 0, 0, 0, 0, 0, 0, 30, 0, + 0, 0, 2, 0, 0, 0, 1, 97, 0, 0, 0, 20, 0, 0, 0, 20, 0, 0, + 0, 1, 98, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; + + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); // skip version ASSERT_OK(input_stream->Seek(1, SeekOrigin::FS_SEEK_SET)); BitmapFileIndexMetaV2 index_meta(FieldType::STRING, index_bytes.size(), GetDefaultPool()); @@ -101,15 +104,15 @@ TEST(BitmapFileIndexMetaTest, TestStringType) { } TEST(BitmapFileIndexMetaTest, TestInvalidType) { - std::vector index_bytes = { - 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, - 1, 0, 0, 0, 1, 97, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 2, 0, 0, 0, - 1, 97, 0, 0, 0, 20, 0, 0, 0, 20, 0, 0, 0, 1, 98, -1, -1, -1, -3, -1, -1, - -1, -1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, - 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; - - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, + 1, 0, 0, 0, 1, 97, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 2, 0, 0, 0, + 1, 97, 0, 0, 0, 20, 0, 0, 0, 20, 0, 0, 0, 1, 98, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, + 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; + + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); // skip version ASSERT_OK(input_stream->Seek(1, SeekOrigin::FS_SEEK_SET)); BitmapFileIndexMetaV2 index_meta(FieldType::DECIMAL, index_bytes.size(), GetDefaultPool()); diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp index cbe92a8b..d2bd4596 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp @@ -18,7 +18,10 @@ #include "paimon/common/file_index/bitmap/bitmap_file_index.h" +#include +#include #include +#include #include "arrow/api.h" #include "arrow/c/bridge.h" @@ -35,6 +38,17 @@ #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +const char* AsCharData(const std::vector& bytes) { + return reinterpret_cast(bytes.data()); +} + +std::vector ToBytes(const char* data, std::size_t size) { + return std::vector(data, data + size); +} + +} // namespace class BitmapIndexTest : public ::testing::Test { public: void SetUp() override { @@ -135,22 +149,24 @@ TEST_F(BitmapIndexTest, TestStringType) { } { // test v1 version - std::vector index_bytes = { - 1, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, 97, 0, 0, 0, 20, 0, - 0, 0, 1, 98, -1, -1, -1, -3, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, - 1, 0, 3, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; + std::vector index_bytes = { + 1, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, + 97, 0, 0, 0, 20, 0, 0, 0, 1, 98, 0xff, 0xff, 0xff, 0xfd, 58, 48, 0, 0, + 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, + 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; - check_result(index_bytes.data(), index_bytes.size()); + check_result(AsCharData(index_bytes), index_bytes.size()); } { // test v2 version - std::vector index_bytes = { - 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, - 1, 0, 0, 0, 1, 97, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 2, 0, 0, 0, - 1, 97, 0, 0, 0, 20, 0, 0, 0, 20, 0, 0, 0, 1, 98, -1, -1, -1, -3, -1, -1, - -1, -1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, - 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 20, + 0, 0, 0, 1, 0, 0, 0, 1, 97, 0, 0, 0, 0, 0, 0, 0, 30, 0, + 0, 0, 2, 0, 0, 0, 1, 97, 0, 0, 0, 20, 0, 0, 0, 20, 0, 0, + 0, 1, 98, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 4, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); } } @@ -198,20 +214,21 @@ TEST_F(BitmapIndexTest, TestBooleanType) { } { // test v1 version - std::vector index_bytes = {1, 0, 0, 0, 5, 0, 0, 0, 2, 1, -1, -1, -1, -5, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 20, 58, 48, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, 0, 0, - 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 2, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 1, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0xff, 0xff, 0xff, 0xfb, 0, 0, 0, 0, 0, 1, 0, 0, + 0, 20, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, + 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 2, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); } { // test v2 version - std::vector index_bytes = { - 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, -1, -1, -1, -5, 0, 0, 0, 18, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 0, 0, 0, - 20, 0, 0, 0, 20, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, - 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 2, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 5, 0, 0, 0, 2, 1, 0xff, 0xff, 0xff, 0xfb, 0, 0, 0, 18, 0, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 20, 1, 0, 0, 0, 20, 0, 0, 0, 20, 58, 48, 0, 0, + 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 2, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); } } @@ -257,20 +274,20 @@ TEST_F(BitmapIndexTest, TestTinyIntType) { } { // test v1 version - std::vector index_bytes = {1, 0, 0, 0, 5, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, - 0, 0, 22, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 1, 0, 0, 0, 5, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 22, 58, 48, + 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); } { // test v2 version - std::vector index_bytes = { + std::vector index_bytes = { 2, 0, 0, 0, 5, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 1, 1, 0, 0, 0, 22, 0, 0, 0, 20, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0}; - check_result(index_bytes.data(), index_bytes.size()); + check_result(AsCharData(index_bytes), index_bytes.size()); } } @@ -317,27 +334,26 @@ TEST_F(BitmapIndexTest, TestSmallIntType) { // as unique non-null value cardinality = 1, can test compatible with java { // test v1 version - std::vector index_bytes = {1, 0, 0, 0, 5, 0, 0, 0, 1, 1, -1, -1, -1, -1, 0, 1, 0, 0, - 0, 0, 59, 48, 0, 0, 1, 0, 0, 3, 0, 1, 0, 1, 0, 3, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = {1, 0, 0, 0, 5, 0, 0, 0, 1, 1, 0xff, 0xff, + 0xff, 0xff, 0, 1, 0, 0, 0, 0, 59, 48, 0, 0, + 1, 0, 0, 3, 0, 1, 0, 1, 0, 3, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); // test compatible with java ASSERT_OK_AND_ASSIGN(auto index_bytes2, write_data(/*version=*/1)); - ASSERT_EQ(index_bytes, std::vector(index_bytes2->data(), - index_bytes2->data() + index_bytes2->size())); + ASSERT_EQ(index_bytes, ToBytes(index_bytes2->data(), index_bytes2->size())); } { // test v2 version - std::vector index_bytes = {2, 0, 0, 0, 5, 0, 0, 0, 1, 1, -1, -1, -1, -1, 0, 0, - 0, 18, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 15, 59, 48, - 0, 0, 1, 0, 0, 3, 0, 1, 0, 1, 0, 3, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 5, 0, 0, 0, 1, 1, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 18, 0, 0, 0, + 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 15, 59, 48, 0, 0, 1, 0, 0, 3, 0, 1, 0, 1, 0, 3, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); // test compatible with java ASSERT_OK_AND_ASSIGN(auto index_bytes2, write_data(/*version=*/2)); - ASSERT_EQ(index_bytes, std::vector(index_bytes2->data(), - index_bytes2->data() + index_bytes2->size())); + ASSERT_EQ(index_bytes, ToBytes(index_bytes2->data(), index_bytes2->size())); } } @@ -382,24 +398,25 @@ TEST_F(BitmapIndexTest, TestBigIntType) { } { // test v1 version - std::vector index_bytes = {1, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, - 0, 1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 2, -1, -1, - -1, -2, 0, 0, 0, 0, 0, 0, 0, 3, -1, -1, -1, -3}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = {1, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0xff, 0xff, 0xff, 0xff, 0, 0, + 0, 0, 0, 0, 0, 2, 0xff, 0xff, 0xff, 0xfe, 0, 0, + 0, 0, 0, 0, 0, 3, 0xff, 0xff, 0xff, 0xfd}; + check_result(AsCharData(index_bytes), index_bytes.size()); } { // test v2 version - std::vector index_bytes = { - 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, - -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 2, -1, -1, -1, -2, -1, - -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 3, -1, -1, -1, -3, -1, -1, -1, -1}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 3, + 0, 0, 0, 0, 0, 0, 0, 1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, + 0, 0, 0, 0, 0, 0, 2, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff, 0, 0, + 0, 0, 0, 0, 0, 3, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff}; + check_result(AsCharData(index_bytes), index_bytes.size()); // test compatible with java ASSERT_OK_AND_ASSIGN(auto index_bytes2, write_data(/*version=*/2)); - ASSERT_EQ(index_bytes, std::vector(index_bytes2->data(), - index_bytes2->data() + index_bytes2->size())); + ASSERT_EQ(index_bytes, ToBytes(index_bytes2->data(), index_bytes2->size())); } } @@ -448,23 +465,23 @@ TEST_F(BitmapIndexTest, TestDateType) { } { // test v1 version - std::vector index_bytes = { + std::vector index_bytes = { 1, 0, 0, 0, 6, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1, 52, 59, 28, 0, 0, 0, 20, 1, - 52, -119, 62, 0, 0, 0, 40, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, + 52, 0x89, 62, 0, 0, 0, 40, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 2, 0, 3, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 4, 0, 5, 0}; - check_result(index_bytes.data(), index_bytes.size()); + check_result(AsCharData(index_bytes), index_bytes.size()); } { // test v2 version - std::vector index_bytes = { + std::vector index_bytes = { 2, 0, 0, 0, 6, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1, 1, 52, 59, 28, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 2, 1, 52, 59, 28, - 0, 0, 0, 20, 0, 0, 0, 20, 1, 52, -119, 62, 0, 0, 0, 40, 0, 0, 0, 20, 58, + 0, 0, 0, 20, 0, 0, 0, 20, 1, 52, 0x89, 62, 0, 0, 0, 40, 0, 0, 0, 20, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 2, 0, 3, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 4, 0, 5, 0}; - check_result(index_bytes.data(), index_bytes.size()); + check_result(AsCharData(index_bytes), index_bytes.size()); } } @@ -511,17 +528,19 @@ TEST_F(BitmapIndexTest, TestIntType) { } { // test v1 version - std::vector index_bytes = {1, 0, 0, 0, 3, 0, 0, 0, 2, 1, -1, -1, -1, -3, 0, - 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 1, -1, -1, -1, -2}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 1, 0, 0, 0, 3, 0, 0, 0, 2, 1, 0xff, 0xff, 0xff, 0xfd, 0, + 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 1, 0xff, 0xff, 0xff, 0xfe}; + check_result(AsCharData(index_bytes), index_bytes.size()); } { // test v2 version - std::vector index_bytes = {2, 0, 0, 0, 3, 0, 0, 0, 2, 1, -1, -1, -1, -3, 0, 0, - 0, 18, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 28, 0, 0, 0, 2, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, - -1, -1, 0, 0, 0, 1, -1, -1, -1, -2, -1, -1, -1, -1}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 3, 0, 0, 0, 2, 1, 0xff, 0xff, 0xff, 0xfd, 0, 0, + 0, 18, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 28, 0, 0, 0, 2, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0, 0, 0, 1, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff}; + check_result(AsCharData(index_bytes), index_bytes.size()); } } @@ -594,28 +613,33 @@ TEST_F(BitmapIndexTest, TestTimestampType) { } { // test v1 version - std::vector index_bytes = { - 1, 0, 0, 0, 8, 0, 0, 0, 6, 1, -1, -1, -1, -6, 0, 6, 51, -113, - -38, -89, 72, -5, 0, 0, 0, 0, -1, -1, -1, -1, -1, -27, -82, 51, -1, -1, - -1, -8, -1, -1, -1, -1, -1, -27, 17, -13, -1, -1, -1, -5, -1, -1, -1, -1, - -1, -27, 96, 19, -1, -1, -1, -4, 0, 6, 51, -113, -50, -69, -122, -5, -1, -1, - -1, -3, 0, 6, 51, -113, -32, -99, 41, -5, -1, -1, -1, -2, 58, 48, 0, 0, - 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 6, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 1, 0, 0, 0, 8, 0, 0, 0, 6, 1, 0xff, 0xff, 0xff, 0xfa, + 0, 6, 51, 0x8f, 0xda, 0xa7, 72, 0xfb, 0, 0, 0, 0, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xe5, 0xae, 51, 0xff, 0xff, 0xff, 0xf8, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xe5, 17, 0xf3, 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, + 96, 19, 0xff, 0xff, 0xff, 0xfc, 0, 6, 51, 0x8f, 0xce, 0xbb, 0x86, 0xfb, + 0xff, 0xff, 0xff, 0xfd, 0, 6, 51, 0x8f, 0xe0, 0x9d, 41, 0xfb, 0xff, 0xff, + 0xff, 0xfe, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 6, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); } { // test v2 version - std::vector index_bytes = { - 2, 0, 0, 0, 8, 0, 0, 0, 6, 1, -1, -1, -1, -6, 0, 0, 0, 18, - 0, 0, 0, 1, -1, -1, -1, -1, -1, -27, 17, -13, 0, 0, 0, 0, 0, 0, - 0, 100, 0, 0, 0, 6, -1, -1, -1, -1, -1, -27, 17, -13, -1, -1, -1, -5, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -27, 96, 19, -1, -1, -1, -4, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -27, -82, 51, -1, -1, -1, -8, -1, -1, -1, -1, - 0, 6, 51, -113, -50, -69, -122, -5, -1, -1, -1, -3, -1, -1, -1, -1, 0, 6, - 51, -113, -38, -89, 72, -5, 0, 0, 0, 0, 0, 0, 0, 20, 0, 6, 51, -113, - -32, -99, 41, -5, -1, -1, -1, -2, -1, -1, -1, -1, 58, 48, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 6, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 2, 0, 0, 0, 8, 0, 0, 0, 6, 1, 0xff, 0xff, 0xff, 0xfa, + 0, 0, 0, 18, 0, 0, 0, 1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, + 17, 0xf3, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 6, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 17, 0xf3, 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 96, 19, 0xff, 0xff, 0xff, 0xfc, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 0xae, 51, 0xff, 0xff, + 0xff, 0xf8, 0xff, 0xff, 0xff, 0xff, 0, 6, 51, 0x8f, 0xce, 0xbb, 0x86, 0xfb, + 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0, 6, 51, 0x8f, 0xda, 0xa7, + 72, 0xfb, 0, 0, 0, 0, 0, 0, 0, 20, 0, 6, 51, 0x8f, + 0xe0, 0x9d, 41, 0xfb, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xff, 0xff, 58, 48, + 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 6, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); } } @@ -731,25 +755,23 @@ TEST_F(BitmapIndexTest, TestCompatibleWithJava) { }; { // test v1 version - std::vector java_index_bytes = { + std::vector java_index_bytes = { 1, 0, 0, 0, 5, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 97, 112, 112, 108, 101, 0, 0, 0, 20, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 4, 0}; ASSERT_OK_AND_ASSIGN(auto index_bytes, write_data(/*version=*/1)); - ASSERT_EQ(java_index_bytes, std::vector(index_bytes->data(), - index_bytes->data() + index_bytes->size())); + ASSERT_EQ(java_index_bytes, ToBytes(index_bytes->data(), index_bytes->size())); } { // test v2 version - std::vector java_index_bytes = { + std::vector java_index_bytes = { 2, 0, 0, 0, 5, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1, 0, 0, 0, 5, 97, 112, 112, 108, 101, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1, 0, 0, 0, 5, 97, 112, 112, 108, 101, 0, 0, 0, 20, 0, 0, 0, 22, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 4, 0}; ASSERT_OK_AND_ASSIGN(auto index_bytes, write_data(/*version=*/2)); - ASSERT_EQ(java_index_bytes, std::vector(index_bytes->data(), - index_bytes->data() + index_bytes->size())); + ASSERT_EQ(java_index_bytes, ToBytes(index_bytes->data(), index_bytes->size())); } } @@ -793,26 +815,24 @@ TEST_F(BitmapIndexTest, TestAllNull) { } { // test v1 version - std::vector index_bytes = {1, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 59, - 48, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 3, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = {1, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 59, + 48, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 3, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); // test compatible ASSERT_OK_AND_ASSIGN(auto index_bytes2, write_data(/*version=*/1)); - ASSERT_EQ(index_bytes, std::vector(index_bytes2->data(), - index_bytes2->data() + index_bytes2->size())); + ASSERT_EQ(index_bytes, ToBytes(index_bytes2->data(), index_bytes2->size())); } { // test v2 version - std::vector index_bytes = {2, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 59, 48, - 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 3, 0}; - check_result(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = {2, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 59, 48, + 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 3, 0}; + check_result(AsCharData(index_bytes), index_bytes.size()); // test compatible ASSERT_OK_AND_ASSIGN(auto index_bytes2, write_data(/*version=*/2)); - ASSERT_EQ(index_bytes, std::vector(index_bytes2->data(), - index_bytes2->data() + index_bytes2->size())); + ASSERT_EQ(index_bytes, ToBytes(index_bytes2->data(), index_bytes2->size())); } } diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp index b678ba3f..c85bffd8 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp @@ -18,6 +18,7 @@ #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h" +#include #include #include "gtest/gtest.h" @@ -65,7 +66,7 @@ class BitSliceIndexBitmapIndexReaderTest : public ::testing::Test { TEST_F(BitSliceIndexBitmapIndexReaderTest, TestMix) { // data: 1, 2, null, -2, -2, -1, null, 2, 0, 5, null - std::vector index_bytes = { + std::vector index_bytes = { 1, 0, 0, 0, 11, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 16, 0, 0, 0, 0, 0, 1, 0, 7, 0, 8, 0, 9, 0, 0, 0, 0, 3, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 9, 0, 58, 48, @@ -74,8 +75,8 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestMix) { 0, 0, 0, 2, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 5, 0, 0, 0, 0, 2, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 5, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0}; - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); BitSliceIndexBitmapFileIndex file_index({}); ASSERT_OK_AND_ASSIGN( auto reader, @@ -120,14 +121,14 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestMix) { TEST_F(BitSliceIndexBitmapIndexReaderTest, TestPositiveOnly) { // data: 0, 1, null, 3, 4, 5, 6, 0, null - std::vector index_bytes = { + std::vector index_bytes = { 1, 0, 0, 0, 9, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 59, 48, 0, 0, 1, 0, 0, 6, 0, 2, 0, 0, 0, 1, 0, 3, 0, 4, 0, 0, 0, 0, 3, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 1, 0, 3, 0, 5, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 4, 0, 5, 0, 6, 0, 0}; - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); BitSliceIndexBitmapFileIndex file_index({}); ASSERT_OK_AND_ASSIGN( auto reader, @@ -171,14 +172,14 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestPositiveOnly) { TEST_F(BitSliceIndexBitmapIndexReaderTest, TestNegativeOnly) { // data: null, -1, null, -3, -4, -5, -6, -1, null - std::vector index_bytes = { + std::vector index_bytes = { 1, 0, 0, 0, 9, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 59, 48, 0, 0, 1, 0, 0, 5, 0, 2, 0, 1, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 3, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 1, 0, 3, 0, 5, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 4, 0, 5, 0, 6, 0}; - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); BitSliceIndexBitmapFileIndex file_index({}); ASSERT_OK_AND_ASSIGN( auto reader, @@ -224,15 +225,15 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestNegativeOnly) { TEST_F(BitSliceIndexBitmapIndexReaderTest, TestPrimitiveType) { // data: null, 1, null, 2, -1 auto check_result = [&](const std::shared_ptr& type, const Literal& literal) { - std::vector index_bytes = { + std::vector index_bytes = { 1, 0, 0, 0, 5, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 2, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0, 0, 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0}; - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); BitSliceIndexBitmapFileIndex file_index({}); ASSERT_OK_AND_ASSIGN(auto reader, file_index.CreateReader(CreateArrowSchema(type).get(), @@ -256,75 +257,81 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestPrimitiveType) { } TEST_F(BitSliceIndexBitmapIndexReaderTest, TestTimestampType) { - std::vector index_bytes = { - 1, 0, 0, 0, 8, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 51, -113, -32, -99, 41, - -5, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, - 0, 6, 0, 0, 0, 0, 51, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, - 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, - 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, - 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, - 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, - 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, - 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 2, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 2, 0, 58, 48, 0, 0, 1, - 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 1, 0, 6, 0, 58, 48, 0, 0, 0, - 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 6, 0, 58, 48, 0, - 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, - 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, - 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, - 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 1, 0, 6, 0, 58, 48, 0, 0, 1, - 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, - 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 6, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, - 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, - 0, 0, 0, 0, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, - 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, - 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, - 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, - 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, - 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, - 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, - 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, - 0, 0, 0, 0, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, - 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, - 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, - 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, - 0, 0, 0, 0, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, - 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, - 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 0, - 0, 0, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, - 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, - 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, -18, 13, 58, 48, 0, - 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, 0, 0, 0, 0, - 21, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, - 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, - 0, 0, 0, 3, 0, 4, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, - 0, 0, 0, 3, 0, 4, 0, 7, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, - 0, 1, 0, 16, 0, 0, 0, 3, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, - 0, 16, 0, 0, 0, 3, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, - 0, 0, 0, 3, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, - 0, 3, 0, 4, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, - 0, 4, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, - 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 7, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0, 58, 48, 0, 0, 1, - 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 4, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, - 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, 0, 58, - 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, - 0, 3, 0, 4, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, - 0, 3, 0, 4, 0, 7, 0}; - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + std::vector index_bytes = { + 1, 0, 0, 0, 8, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 51, 0x8f, 0xe0, + 0x9d, 41, 0xfb, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, + 0, 1, 0, 2, 0, 6, 0, 0, 0, 0, 51, 58, 48, 0, 0, 1, 0, 0, 0, 0, + 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, + 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, + 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, + 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, + 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, + 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, + 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, + 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 2, + 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 1, + 0, 6, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, + 0, 16, 0, 0, 0, 0, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 16, 0, 0, 0, 2, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, + 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, + 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 1, 0, 6, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 2, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 2, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 6, 0, 58, 48, 0, + 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, + 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 6, 0, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 2, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 0, 0, 2, 0, 6, 0, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 6, 0, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, + 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, + 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, + 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, + 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, + 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, + 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, + 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, + 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, + 0, 0, 0, 0, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, + 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, + 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 16, 0, 0, + 0, 0, 0, 1, 0, 2, 0, 6, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 3, + 0, 16, 0, 0, 0, 0, 0, 1, 0, 2, 0, 6, 0, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 26, 0xee, 13, 58, 48, 0, 0, 1, 0, 0, 0, 0, + 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, 0, 0, 0, 0, 21, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, 0, 58, + 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, + 0, 0, 0, 3, 0, 4, 0, 7, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, + 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 7, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 7, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 7, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 7, 0, 58, 48, 0, + 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 4, 0, 7, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 3, 0, 4, 0, 58, 48, 0, 0, 0, + 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, + 0, 4, 0, 7, 0, 58, 48, 0, 0, 0, 0, 0, 0, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, 0, 58, 48, 0, 0, 1, + 0, 0, 0, 0, 0, 2, 0, 16, 0, 0, 0, 3, 0, 4, 0, 7, 0}; + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); BitSliceIndexBitmapFileIndex file_index({}); ASSERT_OK_AND_ASSIGN( auto reader, @@ -371,15 +378,15 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestTimestampType) { } TEST_F(BitSliceIndexBitmapIndexReaderTest, TestUnInvalidType) { - std::vector index_bytes = { + std::vector index_bytes = { 1, 0, 0, 0, 5, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 2, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0, 0, 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0}; - auto input_stream = - std::make_shared(index_bytes.data(), index_bytes.size()); + auto input_stream = std::make_shared( + reinterpret_cast(index_bytes.data()), index_bytes.size()); BitSliceIndexBitmapFileIndex file_index({}); ASSERT_NOK_WITH_MSG( file_index.CreateReader(CreateArrowSchema(arrow::boolean()).get(), diff --git a/src/paimon/common/file_index/file_index_format_test.cpp b/src/paimon/common/file_index/file_index_format_test.cpp index 7851d57e..4ea7863c 100644 --- a/src/paimon/common/file_index/file_index_format_test.cpp +++ b/src/paimon/common/file_index/file_index_format_test.cpp @@ -17,6 +17,7 @@ */ #include "paimon/file_index/file_index_format.h" +#include #include #include "gtest/gtest.h" @@ -57,12 +58,12 @@ class FileIndexFormatTest : public ::testing::Test { TEST_F(FileIndexFormatTest, TestCreateEmptyFileIndexReader) { auto schema = arrow::schema({arrow::field("c1", arrow::utf8())}); - std::vector index_file_bytes = {0, 5, 78, 78, -48, 26, 53, -82, 0, 0, 0, 1, - 0, 0, 0, 47, 0, 0, 0, 1, 0, 2, 99, 49, - 0, 0, 0, 1, 0, 5, 101, 109, 112, 116, 121, -1, - -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0}; - auto input_stream = - std::make_shared(index_file_bytes.data(), index_file_bytes.size()); + std::vector index_file_bytes = { + 0, 5, 78, 78, 0xd0, 26, 53, 0xae, 0, 0, 0, 1, 0, 0, 0, 47, + 0, 0, 0, 1, 0, 2, 99, 49, 0, 0, 0, 1, 0, 5, 101, 109, + 112, 116, 121, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0}; + auto input_stream = std::make_shared( + reinterpret_cast(index_file_bytes.data()), index_file_bytes.size()); ASSERT_OK_AND_ASSIGN(auto reader, FileIndexFormat::CreateReader(input_stream, pool_)); ASSERT_OK_AND_ASSIGN(auto index_file_readers, reader->ReadColumnIndex("c1", CreateArrowSchema(schema).get())); diff --git a/src/paimon/common/io/data_input_output_stream_test.cpp b/src/paimon/common/io/data_input_output_stream_test.cpp index 4e606370..fe186a7e 100644 --- a/src/paimon/common/io/data_input_output_stream_test.cpp +++ b/src/paimon/common/io/data_input_output_stream_test.cpp @@ -46,25 +46,25 @@ class DataInputOutputStreamTest : public ::testing::Test, protected: void SetUp() override { pool_ = GetDefaultPool(); - std::vector bytes_big_endian = { - 127, 125, 67, 127, -2, -57, 127, 127, -1, -1, -1, -1, -1, -1, - -3, 1, 0, 39, 84, 104, 105, 115, 32, 105, 115, 32, 97, 32, - 118, 101, 114, 121, 32, 118, 101, 114, 121, 32, 118, 101, 114, 121, - 32, 108, 111, 110, 103, 32, 115, 101, 110, 116, 101, 110, 99, 101, - 46, -26, -120, -111, -26, -104, -81, -28, -72, -128, -28, -72, -86, -25, - -78, -119, -27, -120, -73, -27, -116, -96, -17, -67, -98}; + std::vector bytes_big_endian = { + 127, 125, 67, 127, 0xfe, 0xc7, 127, 127, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfd, 1, 0, 39, 84, 104, 105, 115, 32, 105, 115, 32, 97, 32, + 118, 101, 114, 121, 32, 118, 101, 114, 121, 32, 118, 101, 114, 121, + 32, 108, 111, 110, 103, 32, 115, 101, 110, 116, 101, 110, 99, 101, + 46, 0xe6, 0x88, 0x91, 0xe6, 0x98, 0xaf, 0xe4, 0xb8, 0x80, 0xe4, 0xb8, 0xaa, 0xe7, + 0xb2, 0x89, 0xe5, 0x88, 0xb7, 0xe5, 0x8c, 0xa0, 0xef, 0xbd, 0x9e}; serialized_bytes_big_endian_ = std::make_shared(bytes_big_endian.size(), pool_.get()); memcpy(serialized_bytes_big_endian_->data(), bytes_big_endian.data(), bytes_big_endian.size()); - std::vector bytes_little_endian = { - 127, 67, 125, 127, -57, -2, 127, -3, -1, -1, -1, -1, -1, -1, - 127, 1, 39, 0, 84, 104, 105, 115, 32, 105, 115, 32, 97, 32, - 118, 101, 114, 121, 32, 118, 101, 114, 121, 32, 118, 101, 114, 121, - 32, 108, 111, 110, 103, 32, 115, 101, 110, 116, 101, 110, 99, 101, - 46, -26, -120, -111, -26, -104, -81, -28, -72, -128, -28, -72, -86, -25, - -78, -119, -27, -120, -73, -27, -116, -96, -17, -67, -98}; + std::vector bytes_little_endian = { + 127, 67, 125, 127, 0xc7, 0xfe, 127, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 127, 1, 39, 0, 84, 104, 105, 115, 32, 105, 115, 32, 97, 32, + 118, 101, 114, 121, 32, 118, 101, 114, 121, 32, 118, 101, 114, 121, + 32, 108, 111, 110, 103, 32, 115, 101, 110, 116, 101, 110, 99, 101, + 46, 0xe6, 0x88, 0x91, 0xe6, 0x98, 0xaf, 0xe4, 0xb8, 0x80, 0xe4, 0xb8, 0xaa, 0xe7, + 0xb2, 0x89, 0xe5, 0x88, 0xb7, 0xe5, 0x8c, 0xa0, 0xef, 0xbd, 0x9e}; serialized_bytes_little_endian_ = std::make_shared(bytes_little_endian.size(), pool_.get()); memcpy(serialized_bytes_little_endian_->data(), bytes_little_endian.data(), diff --git a/src/paimon/common/utils/binary_row_partition_computer_test.cpp b/src/paimon/common/utils/binary_row_partition_computer_test.cpp index 0bbe5e27..e02fcc14 100644 --- a/src/paimon/common/utils/binary_row_partition_computer_test.cpp +++ b/src/paimon/common/utils/binary_row_partition_computer_test.cpp @@ -77,7 +77,7 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) { ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map)); ASSERT_EQ(12, row.GetFieldCount()); ASSERT_EQ(true, row.GetBoolean(0)); - ASSERT_EQ(-20, row.GetByte(1)); + ASSERT_EQ(-20, static_cast(row.GetByte(1))); ASSERT_EQ(10, row.GetByte(2)); ASSERT_EQ(1556, row.GetShort(3)); ASSERT_EQ(-2556, row.GetShort(4)); @@ -121,7 +121,7 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) { ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map)); ASSERT_EQ(12, row.GetFieldCount()); ASSERT_EQ(true, row.GetBoolean(0)); - ASSERT_EQ(-20, row.GetByte(1)); + ASSERT_EQ(-20, static_cast(row.GetByte(1))); ASSERT_EQ(10, row.GetByte(2)); ASSERT_EQ(1556, row.GetShort(3)); ASSERT_EQ(-2556, row.GetShort(4)); @@ -165,7 +165,7 @@ TEST(BinaryRowPartitionComputerTest, TestToAndFromBinaryRow) { ASSERT_OK_AND_ASSIGN(BinaryRow row, computer->ToBinaryRow(partition_map)); ASSERT_EQ(12, row.GetFieldCount()); ASSERT_EQ(true, row.GetBoolean(0)); - ASSERT_EQ(-20, row.GetByte(1)); + ASSERT_EQ(-20, static_cast(row.GetByte(1))); ASSERT_EQ(10, row.GetByte(2)); ASSERT_EQ(1556, row.GetShort(3)); ASSERT_EQ(-2556, row.GetShort(4)); diff --git a/src/paimon/common/utils/data_converter_utils_test.cpp b/src/paimon/common/utils/data_converter_utils_test.cpp index b0078552..f232cbd7 100644 --- a/src/paimon/common/utils/data_converter_utils_test.cpp +++ b/src/paimon/common/utils/data_converter_utils_test.cpp @@ -20,6 +20,7 @@ #include "paimon/common/utils/data_converter_utils.h" #include +#include #include #include @@ -71,7 +72,7 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithLegacyPartitionName ASSERT_EQ(data.size(), row.GetFieldCount()); ASSERT_EQ(true, row.GetBoolean(0)); ASSERT_EQ(10, row.GetByte(1)); - ASSERT_EQ(-20, row.GetByte(2)); + ASSERT_EQ(-20, static_cast(row.GetByte(2))); ASSERT_EQ(1556, row.GetShort(3)); ASSERT_EQ(-2556, row.GetShort(4)); ASSERT_EQ(348489, row.GetInt(5)); @@ -125,7 +126,7 @@ TEST(DataConverterUtilsTest, TestDataToBinaryRowConverterWithNoLegacyPartitionNa ASSERT_EQ(data.size(), row.GetFieldCount()); ASSERT_EQ(true, row.GetBoolean(0)); ASSERT_EQ(10, row.GetByte(1)); - ASSERT_EQ(-20, row.GetByte(2)); + ASSERT_EQ(-20, static_cast(row.GetByte(2))); ASSERT_EQ(1556, row.GetShort(3)); ASSERT_EQ(-2556, row.GetShort(4)); ASSERT_EQ(348489, row.GetInt(5)); diff --git a/src/paimon/common/utils/murmurhash_utils.h b/src/paimon/common/utils/murmurhash_utils.h index 52831c9a..69fa0d6f 100644 --- a/src/paimon/common/utils/murmurhash_utils.h +++ b/src/paimon/common/utils/murmurhash_utils.h @@ -184,7 +184,7 @@ class MurmurHashUtils { int32_t length_aligned = length_in_bytes - length_in_bytes % 4; int32_t h1 = HashBytesByInt(segment, offset, length_aligned, seed); for (int32_t i = length_aligned; i < length_in_bytes; i++) { - int32_t k1 = MixK1(segment.Get(offset + i)); + int32_t k1 = MixK1(static_cast(segment.Get(offset + i))); h1 = MixH1(h1, k1); } return Fmix(h1, length_in_bytes); @@ -240,9 +240,9 @@ class MurmurHashUtils { return value; } - static char GetByte(const void* base, int64_t offset) { - char value; - std::memcpy(&value, static_cast(base) + offset, sizeof(char)); + static std::int8_t GetByte(const void* base, int64_t offset) { + std::int8_t value; + std::memcpy(&value, static_cast(base) + offset, sizeof(std::int8_t)); return value; } From 52c866ac136838a0a8559bbb426fe5eed52026ad Mon Sep 17 00:00:00 2001 From: kid <19265318+u70b3@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:11:37 +0000 Subject: [PATCH 2/2] Preserve BinaryString byte classifier ABI --- src/paimon/common/data/binary_string.cpp | 11 ++++++----- src/paimon/common/data/binary_string.h | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/paimon/common/data/binary_string.cpp b/src/paimon/common/data/binary_string.cpp index 51457e98..47e93998 100644 --- a/src/paimon/common/data/binary_string.cpp +++ b/src/paimon/common/data/binary_string.cpp @@ -74,17 +74,18 @@ std::string BinaryString::ToString() const { return ret; } -int32_t BinaryString::NumBytesForFirstByte(std::uint8_t b) { - if ((b & 0x80) == 0) { +int32_t BinaryString::NumBytesForFirstByte(char b) { + const auto byte = static_cast(b); + if ((byte & 0x80) == 0) { // 1 byte, 7 bits: 0xxxxxxx return 1; - } else if ((b & 0xe0) == 0xc0 && (b & 0x1e) != 0) { + } else if ((byte & 0xe0) == 0xc0 && (byte & 0x1e) != 0) { // 2 bytes, 11 bits: 110xxxxx 10xxxxxx return 2; - } else if ((b & 0xf0) == 0xe0) { + } else if ((byte & 0xf0) == 0xe0) { // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx return 3; - } else if ((b & 0xf8) == 0xf0) { + } else if ((byte & 0xf8) == 0xf0) { // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx return 4; } else { diff --git a/src/paimon/common/data/binary_string.h b/src/paimon/common/data/binary_string.h index 57c80f83..2de9b588 100644 --- a/src/paimon/common/data/binary_string.h +++ b/src/paimon/common/data/binary_string.h @@ -138,7 +138,7 @@ class PAIMON_EXPORT BinaryString : public BinarySection { /// @return the number of bytes for a code point with the first byte as b. /// @param b The first byte of a code point - static int32_t NumBytesForFirstByte(std::uint8_t b); + static int32_t NumBytesForFirstByte(char b); private: char GetByteOneSegment(int32_t i) const;