From e4e284997fe54d91b5250020a868b6f92939be9f Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Thu, 5 Nov 2026 13:50:36 +0000 Subject: [PATCH 01/19] LOGBROKER-10371 Add consumer offsets mirroring (#57493) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 4 + .../persqueue_public/impl/persqueue.cpp | 4 + .../persqueue_public/impl/persqueue_impl.h | 2 + .../persqueue_public/include/control_plane.h | 8 + .../replication_attributes_ut.cpp | 88 ++++++++ src/client/topic/common/replication.h | 23 ++ src/client/topic/impl/read_session_impl.ipp | 29 ++- .../read_session_replication_ut.cpp | 197 ++++++++++++++++++ 9 files changed, 347 insertions(+), 10 deletions(-) create mode 100644 src/client/persqueue_public/ut/control_plane/replication_attributes_ut.cpp create mode 100644 src/client/topic/common/replication.h create mode 100644 src/client/topic/ut/replication/read_session_replication_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index e31bfe03864..2d351f744f1 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -e298c2650fafe1aab48d99022912303996c9e710 +805b685e01e316136b6b022252003b19cbeab663 diff --git a/CHANGELOG.md b/CHANGELOG.md index a7e7ba85cb9..9e0aa36209d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +* Fixed PersQueue topic settings losing replication-consumer and consumer-offset-mirroring attributes when copied from DescribeTopic into CreateTopic or AlterTopic settings. + +* Supported Topic read sessions counting replication control messages as data offsets, which could block commits after an offset gap. + ## v3.26.0 * Federated topic read sessions now follow federation membership. After the initial discovery, databases that join the federation are read and databases that leave it are dropped; partition sessions of a dropped database get `TPartitionSessionClosedEvent` with the `ConnectionLost` reason. A listed database that becomes unavailable keeps its subsession. The refresh period is `TFederatedTopicClientSettings::FederationDiscoveryInterval` (30 seconds by default). diff --git a/src/client/persqueue_public/impl/persqueue.cpp b/src/client/persqueue_public/impl/persqueue.cpp index 0188c5d749e..c01379e925f 100644 --- a/src/client/persqueue_public/impl/persqueue.cpp +++ b/src/client/persqueue_public/impl/persqueue.cpp @@ -165,6 +165,10 @@ TDescribeTopicResult::TTopicSettings::TTopicSettings(const Ydb::PersQueue::V1::T FederationAccount_ = pair.second; } else if (pair.first == "_advanced_monitoring") { AdvancedMonitoringSettings_ = pair.second; + } else if (pair.first == "_ydb_replication_consumer") { + ReplicationConsumer_ = pair.second; + } else if (pair.first == "_ydb_mirror_consumer_offsets") { + MirrorConsumerOffsets_ = !pair.second.empty() && FromString(pair.second); } } for (const auto& readRule : settings.read_rules()) { diff --git a/src/client/persqueue_public/impl/persqueue_impl.h b/src/client/persqueue_public/impl/persqueue_impl.h index 18ecfa2cdf0..5039c13147d 100644 --- a/src/client/persqueue_public/impl/persqueue_impl.h +++ b/src/client/persqueue_public/impl/persqueue_impl.h @@ -145,6 +145,8 @@ class TPersQueueClient::TImpl : public TClientImplCommon, FederationAccount); GETTER(std::optional, MetricsLevel); GETTER(std::optional, AdvancedMonitoringSettings); + GETTER(std::optional, ReplicationConsumer); + GETTER(std::optional, MirrorConsumerOffsets); const std::vector& ReadRules() const { return ReadRules_; @@ -199,6 +201,8 @@ struct TDescribeTopicResult : public TStatus { std::string FederationAccount_; std::optional MetricsLevel_; std::optional AdvancedMonitoringSettings_; + std::optional ReplicationConsumer_; + std::optional MirrorConsumerOffsets_; std::optional MaxPartitionsCount_; std::optional StabilizationWindow_; @@ -313,6 +317,8 @@ struct TTopicSettings : public TOperationRequestSettings { FLUENT_SETTING_OPTIONAL(std::string, FederationAccount); FLUENT_SETTING_OPTIONAL(ui32, MetricsLevel); FLUENT_SETTING_OPTIONAL(std::string, AdvancedMonitoringSettings); + FLUENT_SETTING_OPTIONAL(std::string, ReplicationConsumer); + FLUENT_SETTING_OPTIONAL(bool, MirrorConsumerOffsets); //TODO: FLUENT_SETTING_VECTOR FLUENT_SETTING_DEFAULT(std::vector, ReadRules, {}); @@ -345,6 +351,8 @@ struct TTopicSettings : public TOperationRequestSettings { FederationAccount_ = settings.FederationAccount(); MetricsLevel_ = settings.MetricsLevel(); AdvancedMonitoringSettings_ = settings.AdvancedMonitoringSettings(); + ReplicationConsumer_ = settings.ReplicationConsumer(); + MirrorConsumerOffsets_ = settings.MirrorConsumerOffsets(); ReadRules_.clear(); for (const auto& readRule : settings.ReadRules()) { ReadRules_.push_back({}); diff --git a/src/client/persqueue_public/ut/control_plane/replication_attributes_ut.cpp b/src/client/persqueue_public/ut/control_plane/replication_attributes_ut.cpp new file mode 100644 index 00000000000..9f910d7a4a3 --- /dev/null +++ b/src/client/persqueue_public/ut/control_plane/replication_attributes_ut.cpp @@ -0,0 +1,88 @@ +#include +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + namespace { + + constexpr const char* ReplicationConsumerAttribute = "_ydb_replication_consumer"; + constexpr const char* MirrorConsumerOffsetsAttribute = "_ydb_mirror_consumer_offsets"; + constexpr const char* ConsumerName = "shared/mirror-from-source-to-shadow"; + + template + void TestReplicationAttributesRoundTrip() { + for (const bool enabled : {false, true}) { + Ydb::PersQueue::V1::TopicSettings proto; + (*proto.mutable_attributes())[ReplicationConsumerAttribute] = ConsumerName; + (*proto.mutable_attributes())[MirrorConsumerOffsetsAttribute] = enabled ? "true" : "false"; + const TDescribeTopicResult::TTopicSettings description(proto); + UNIT_ASSERT(description.ReplicationConsumer().has_value()); + UNIT_ASSERT_VALUES_EQUAL(*description.ReplicationConsumer(), ConsumerName); + UNIT_ASSERT(description.MirrorConsumerOffsets().has_value()); + UNIT_ASSERT_VALUES_EQUAL(*description.MirrorConsumerOffsets(), enabled); + + // This is the conversion used by config-manager's Create/Alter executors. + TSettings settings; + settings.SetSettings(description); + const auto request = TPersQueueClient::TImpl::MakePropsCreateOrAlterRequest("topic", settings); + const auto& attributes = request.settings().attributes(); + UNIT_ASSERT_VALUES_EQUAL(attributes.at(ReplicationConsumerAttribute), ConsumerName); + UNIT_ASSERT_VALUES_EQUAL(attributes.at(MirrorConsumerOffsetsAttribute), enabled ? "true" : "false"); + } + } + + template + void TestAbsentReplicationAttributes() { + const TDescribeTopicResult::TTopicSettings description(Ydb::PersQueue::V1::TopicSettings{}); + UNIT_ASSERT(!description.ReplicationConsumer().has_value()); + UNIT_ASSERT(!description.MirrorConsumerOffsets().has_value()); + + TSettings settings; + settings.ReplicationConsumer(ConsumerName).MirrorConsumerOffsets(true); + settings.SetSettings(description); + const auto request = TPersQueueClient::TImpl::MakePropsCreateOrAlterRequest("topic", settings); + UNIT_ASSERT(!request.settings().attributes().contains(ReplicationConsumerAttribute)); + UNIT_ASSERT(!request.settings().attributes().contains(MirrorConsumerOffsetsAttribute)); + } + + } // namespace + + Y_UNIT_TEST_SUITE(TPersQueueReplicationAttributes) { + Y_UNIT_TEST(CreateTopicPreservesReplicationAttributes) { + TestReplicationAttributesRoundTrip(); + } + + Y_UNIT_TEST(AlterTopicPreservesReplicationAttributes) { + TestReplicationAttributesRoundTrip(); + } + + Y_UNIT_TEST(CreateTopicDoesNotIntroduceAbsentReplicationAttributes) { + TestAbsentReplicationAttributes(); + } + + Y_UNIT_TEST(AlterTopicDoesNotIntroduceAbsentReplicationAttributes) { + TestAbsentReplicationAttributes(); + } + + Y_UNIT_TEST(FluentSettingsCanExplicitlyDisableOffsetMirroring) { + TAlterTopicSettings settings; + settings.ReplicationConsumer(ConsumerName).MirrorConsumerOffsets(false); + const auto request = TPersQueueClient::TImpl::MakePropsCreateOrAlterRequest("topic", settings); + UNIT_ASSERT_VALUES_EQUAL(request.settings().attributes().at(ReplicationConsumerAttribute), ConsumerName); + UNIT_ASSERT_VALUES_EQUAL(request.settings().attributes().at(MirrorConsumerOffsetsAttribute), "false"); + } + + Y_UNIT_TEST(EmptyAttributesKeepServerSemantics) { + Ydb::PersQueue::V1::TopicSettings proto; + (*proto.mutable_attributes())[ReplicationConsumerAttribute] = ""; + (*proto.mutable_attributes())[MirrorConsumerOffsetsAttribute] = ""; + TAlterTopicSettings settings; + settings.SetSettings(TDescribeTopicResult::TTopicSettings(proto)); + const auto request = TPersQueueClient::TImpl::MakePropsCreateOrAlterRequest("topic", settings); + UNIT_ASSERT_VALUES_EQUAL(request.settings().attributes().at(ReplicationConsumerAttribute), ""); + UNIT_ASSERT_VALUES_EQUAL(request.settings().attributes().at(MirrorConsumerOffsetsAttribute), "false"); + } + } // Y_UNIT_TEST_SUITE(TPersQueueReplicationAttributes) + +} // namespace NYdb::inline V3::NPersQueue diff --git a/src/client/topic/common/replication.h b/src/client/topic/common/replication.h new file mode 100644 index 00000000000..b9a15369264 --- /dev/null +++ b/src/client/topic/common/replication.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3::NTopic { + + // Metadata is a range of key/value pairs; a projection also supports protobuf items. + template + bool IsReplicationServiceMessage(uint64_t offset, const TMetadata& metadata, TProjection projection = {}) { + if (offset != 0) { + return false; + } + for (const auto& item : metadata) { + const auto& [key, value] = projection(item); + if (key == "_ydb_service_replication_message" && value == "true") { + return true; + } + } + return false; + } + +} // namespace NYdb::inline V3::NTopic diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 5458fe4ba75..c57fe8302ce 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -9,6 +9,7 @@ // #include #include +#include #define INCLUDE_YDB_INTERNAL_H #include @@ -28,6 +29,7 @@ #include #include +#include #include #include @@ -1436,6 +1438,24 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( for (const auto& batch : partitionData.batches()) { // Validate messages. for (const auto& messageData : batch.message_data()) { + if (firstOffset == std::numeric_limits::max()) { + firstOffset = messageData.offset(); + } + const ui64 logicalMessageCount = GetReadMessageCount(messageData, batch.codec()); + const i64 messageSize = static_cast(messageData.data().size()); + CompressedDataSize += messageSize; + *Settings.Counters_->BytesInflightTotal += messageSize; + *Settings.Counters_->BytesInflightCompressed += messageSize; + *Settings.Counters_->MessagesInflight += logicalMessageCount; + + // Deliver control payloads to the client without making their dummy + // offset part of the read position or the ranges awaiting a commit. + if (IsReplicationServiceMessage(messageData.offset(), messageData.metadata_items(), [](const auto& item) { + return std::pair{item.key(), item.value()}; + })) { + continue; + } + // Check offsets continuity. // A message may start below the desired offset when a multi-message batch // is redelivered as a whole blob after restarting from an offset inside it. @@ -1444,18 +1464,9 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( Y_ABORT_UNLESS(res); } - if (firstOffset == std::numeric_limits::max()) { - firstOffset = messageData.offset(); - } currentOffset = messageData.offset(); - const ui64 logicalMessageCount = GetReadMessageCount(messageData, batch.codec()); desiredOffset = Max(desiredOffset, currentOffset + static_cast(logicalMessageCount)); partitionStream->UpdateMaxReadOffset(currentOffset + static_cast(logicalMessageCount) - 1); - const i64 messageSize = static_cast(messageData.data().size()); - CompressedDataSize += messageSize; - *Settings.Counters_->BytesInflightTotal += messageSize; - *Settings.Counters_->BytesInflightCompressed += messageSize; - *Settings.Counters_->MessagesInflight += logicalMessageCount; } } if (firstOffset == std::numeric_limits::max()) { diff --git a/src/client/topic/ut/replication/read_session_replication_ut.cpp b/src/client/topic/ut/replication/read_session_replication_ut.cpp new file mode 100644 index 00000000000..2d6a54059ed --- /dev/null +++ b/src/client/topic/ut/replication/read_session_replication_ut.cpp @@ -0,0 +1,197 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include + +namespace NYdb::inline V3::NTopic::NTests { + namespace { + + constexpr ui64 FirstOffset = 1000; + constexpr ui64 EndOffset = FirstOffset + 2; + const std::string ControlPayload = "replication-offsets"; + + class TReplicationReadService: public Ydb::Topic::V1::TopicService::Service { + using TClientMessage = Ydb::Topic::StreamReadMessage::FromClient; + using TServerMessage = Ydb::Topic::StreamReadMessage::FromServer; + + public: + grpc::Status StreamRead(grpc::ServerContext*, grpc::ServerReaderWriter* stream) override { + TDisjointIntervalTree commits; + TClientMessage request; + while (stream->Read(&request)) { + if (request.has_init_request()) { + TServerMessage response; + response.set_status(Ydb::StatusIds::SUCCESS); + response.mutable_init_response()->set_session_id("replication-test"); + if (!stream->Write(response)) { + break; + } + response.Clear(); + response.set_status(Ydb::StatusIds::SUCCESS); + auto* start = response.mutable_start_partition_session_request(); + start->mutable_partition_session()->set_partition_session_id(1); + start->mutable_partition_session()->set_path("topic"); + start->mutable_partition_offsets()->set_start(FirstOffset); + start->mutable_partition_offsets()->set_end(EndOffset); + if (!stream->Write(response)) { + break; + } + } else if (request.has_start_partition_session_response()) { + // Send only after Confirm, so Confirm cannot reset the SDK's + // expected offset after it has processed the control message. + if (!SendMessage(stream, 0, ControlPayload, true) || !SendMessage(stream, FirstOffset, "first", false) || !SendMessage(stream, 0, ControlPayload, true) || !SendMessage(stream, FirstOffset + 1, std::string(512 * 1024 + 1, 'x'), false)) { + break; + } + } else if (request.has_commit_offset_request()) { + for (const auto& partition : request.commit_offset_request().commit_offsets()) { + for (const auto& range : partition.offsets()) { + commits.InsertInterval(range.start(), range.end()); + } + } + if (!commits.Empty() && commits.Min() == 0 && commits.begin()->second == EndOffset) { + TServerMessage response; + response.set_status(Ydb::StatusIds::SUCCESS); + auto* committed = response.mutable_commit_offset_response()->add_partitions_committed_offsets(); + committed->set_partition_session_id(1); + committed->set_committed_offset(EndOffset); + if (!stream->Write(response)) { + break; + } + } + } + } + return grpc::Status::OK; + } + + private: + static bool SendMessage(grpc::ServerReaderWriter* stream, + ui64 offset, const std::string& payload, bool service) { + TServerMessage response; + response.set_status(Ydb::StatusIds::SUCCESS); + response.mutable_read_response()->set_bytes_size(payload.size()); + auto* partition = response.mutable_read_response()->add_partition_data(); + partition->set_partition_session_id(1); + auto* batch = partition->add_batches(); + batch->set_codec(Ydb::Topic::CODEC_RAW); + auto* message = batch->add_message_data(); + message->set_offset(offset); + message->set_data(payload); + message->set_uncompressed_size(payload.size()); + if (service) { + auto* metadata = message->add_metadata_items(); + metadata->set_key("_ydb_service_replication_message"); + metadata->set_value("true"); + } + return stream->Write(response); + } + }; + + class TTestServer { + public: + explicit TTestServer(TReplicationReadService& service) + : Address("localhost:" + std::to_string(static_cast(Port))) + { + Server = grpc::ServerBuilder() + .AddListeningPort(TString(Address), grpc::InsecureServerCredentials()) + .RegisterService(&service) + .BuildAndStart(); + UNIT_ASSERT(Server); + } + + ~TTestServer() { + Server->Shutdown(std::chrono::system_clock::now()); + Server->Wait(); + } + + NTesting::TPortHolder Port = NTesting::GetFreePort(); + const std::string Address; + std::unique_ptr Server; + }; + + void TestReplicationMessageCommits(bool decompress) { + TReplicationReadService service; + TTestServer server(service); + TDriver driver(TDriverConfig().SetEndpoint(server.Address).SetDatabase("/Root").SetDiscoveryMode(EDiscoveryMode::Off)); + TTopicClient client(driver); + auto reader = client.CreateReadSession(TReadSessionSettings() + .ConsumerName("replicator") + .AppendTopics(TTopicReadSettings("topic")) + .Decompress(decompress) + .RetryPolicy(IRetryPolicy::GetNoRetryPolicy())); + + ui64 committed = 0; + size_t controlCount = 0; + size_t messageCount = 0; + const auto deadline = TInstant::Now() + TDuration::Seconds(10); + while (committed != EndOffset && TInstant::Now() < deadline) { + if (!reader->WaitEvent().Wait(deadline - TInstant::Now())) { + break; + } + auto event = reader->GetEvent(false); + UNIT_ASSERT(event); + if (auto* start = std::get_if(&*event)) { + start->Confirm(0, 0); + } else if (auto* data = std::get_if(&*event)) { + TDeferredCommit commit; + auto process = [&](const auto& message) { + if (message.GetOffset() == 0) { + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), ControlPayload); + const auto& fields = message.GetMessageMeta()->Fields; + UNIT_ASSERT_VALUES_EQUAL(fields.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(fields.front().first, "_ydb_service_replication_message"); + UNIT_ASSERT_VALUES_EQUAL(fields.front().second, "true"); + ++controlCount; + return; + } + UNIT_ASSERT_VALUES_EQUAL(message.GetOffset(), FirstOffset + messageCount); + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messageCount == 0 ? "first" : std::string(512 * 1024 + 1, 'x')); + commit.Add(message.GetPartitionSession(), message.GetOffset(), message.GetOffset() + 1); + ++messageCount; + }; + if (decompress) { + for (const auto& message : data->GetMessages()) { + process(message); + } + } else { + for (const auto& message : data->GetCompressedMessages()) { + process(message); + } + } + commit.Commit(); + } else if (auto* ack = std::get_if(&*event)) { + committed = ack->GetCommittedOffset(); + } else if (auto* closed = std::get_if(&*event)) { + UNIT_FAIL(closed->DebugString()); + } + } + UNIT_ASSERT_VALUES_EQUAL(committed, EndOffset); + UNIT_ASSERT_VALUES_EQUAL(controlCount, 2); + UNIT_ASSERT_VALUES_EQUAL(messageCount, 2); + UNIT_ASSERT(reader->Close(TDuration::Seconds(5))); + } + + } // namespace + + Y_UNIT_TEST_SUITE(TReadSessionReplication) { + Y_UNIT_TEST(ControlMessagesDoNotBlockCompressedMessageCommits) { + TestReplicationMessageCommits(false); + } + + Y_UNIT_TEST(ControlMessagesDoNotBlockDecompressedMessageCommits) { + TestReplicationMessageCommits(true); + } + } // Y_UNIT_TEST_SUITE(TReadSessionReplication) + +} // namespace NYdb::inline V3::NTopic::NTests From 2eca93ea75241755f0e686cdbfc3c1bfdc7c4c02 Mon Sep 17 00:00:00 2001 From: YDBot Date: Thu, 5 Nov 2026 13:50:45 +0000 Subject: [PATCH 02/19] Merge pull request #57788 from ydb-platform/robots/branch-sync/merge-rightlib-261101-0052 --- .github/last_commit.txt | 2 +- library/cpp/blockcodecs/codecs/lz4/lz4.cpp | 3 +++ library/cpp/blockcodecs/codecs_ut.cpp | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 2d351f744f1..ced5ac22881 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -805b685e01e316136b6b022252003b19cbeab663 +e65e902b4372674b57815a2ed0f09a7794a633b5 diff --git a/library/cpp/blockcodecs/codecs/lz4/lz4.cpp b/library/cpp/blockcodecs/codecs/lz4/lz4.cpp index c3876bd68bb..3dca40e32a2 100644 --- a/library/cpp/blockcodecs/codecs/lz4/lz4.cpp +++ b/library/cpp/blockcodecs/codecs/lz4/lz4.cpp @@ -60,6 +60,9 @@ namespace { if (res < 0) { ythrow TDecompressError(res); } + if (static_cast(res) != len) { + ythrow TDecompressError(len, res); + } } static inline TStringBuf DPrefix() { diff --git a/library/cpp/blockcodecs/codecs_ut.cpp b/library/cpp/blockcodecs/codecs_ut.cpp index e167816680e..4ba5ada0538 100644 --- a/library/cpp/blockcodecs/codecs_ut.cpp +++ b/library/cpp/blockcodecs/codecs_ut.cpp @@ -6,6 +6,7 @@ #include #include #include +#include Y_UNIT_TEST_SUITE(TBlockCodecsTest) { using namespace NBlockCodecs; @@ -16,6 +17,24 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { return bb; } + Y_UNIT_TEST(TestLz4SafeRejectsIncorrectDecompressedLength) { + const TString data(4096, 'a'); + for (const auto& name : ListAllCodecs()) { + const auto* codec = Codec(name); + if (!codec->Name().StartsWith("lz4-") || !codec->Name().EndsWith("-safe")) { + continue; + } + + const TString encoded = codec->Encode(data); + UNIT_ASSERT_VALUES_EQUAL(codec->Decode(encoded), data); + for (const ui64 length : {ui64(2048), ui64(32768)}) { + TString corrupt = encoded; + WriteUnaligned(corrupt.begin(), length); + UNIT_ASSERT_EXCEPTION(codec->Decode(corrupt), TDataError); + } + } + } + void TestAllAtOnce(size_t n, size_t m) { TVector datas; From 890759c6e71a1fd0d3f108fe2fd2570aa0ae7c6d Mon Sep 17 00:00:00 2001 From: Sergey M Date: Thu, 5 Nov 2026 13:50:54 +0000 Subject: [PATCH 03/19] Add E4M3 and E5M2 FP8 vector support (#57246) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/table/table.h | 4 ++++ src/api/protos/ydb_table.proto | 2 ++ src/client/table/out.cpp | 4 ++++ src/client/table/table.cpp | 8 +++++++ tests/unit/client/table/table_ut.cpp | 29 ++++++++++++++++++++++++ 6 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index ced5ac22881..0c46d84dfe5 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -e65e902b4372674b57815a2ed0f09a7794a633b5 +e5e6e86d8936ed5351d5db0a05a4c19eb68eeb7b diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index f786960012d..c5584c3371b 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -341,6 +341,8 @@ struct TVectorIndexSettings { Bit, Float16, BFloat16, + Float8E4M3, // E4M3FN + Float8E5M2, }; EMetric Metric = EMetric::Unspecified; @@ -373,6 +375,8 @@ struct TKMeansTreeSettings { Bit, Float16, BFloat16, + Float8E4M3, // E4M3FN + Float8E5M2, }; TVectorIndexSettings Settings; diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 3f30f71871a..36bcc266dfe 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -82,6 +82,8 @@ message VectorIndexSettings { VECTOR_TYPE_FLOAT = 1; VECTOR_TYPE_FLOAT16 = 5; VECTOR_TYPE_BFLOAT16 = 6; + VECTOR_TYPE_FLOAT8_E4M3 = 7; // E4M3FN, no scale factor. + VECTOR_TYPE_FLOAT8_E5M2 = 8; // E5M2, no scale factor. VECTOR_TYPE_UINT8 = 2; VECTOR_TYPE_INT8 = 3; VECTOR_TYPE_BIT = 4; diff --git a/src/client/table/out.cpp b/src/client/table/out.cpp index 7431756fc99..5a7bb48bdca 100644 --- a/src/client/table/out.cpp +++ b/src/client/table/out.cpp @@ -58,6 +58,10 @@ Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings::EVectorType, stream, va return "float16"; case NYdb::NTable::TVectorIndexSettings::EVectorType::BFloat16: return "bfloat16"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Float8E4M3: + return "float8_e4m3"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Float8E5M2: + return "float8_e5m2"; case NYdb::NTable::TVectorIndexSettings::EVectorType::Uint8: return "uint8"; case NYdb::NTable::TVectorIndexSettings::EVectorType::Int8: diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 0f761eaa7cd..dfce425e915 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -2902,6 +2902,10 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const Ydb::Table::VectorInd return EVectorType::Float16; case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_BFLOAT16: return EVectorType::BFloat16; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E4M3: + return EVectorType::Float8E4M3; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E5M2: + return EVectorType::Float8E5M2; case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8: return EVectorType::Uint8; case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT8: @@ -2946,6 +2950,10 @@ void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT16; case EVectorType::BFloat16: return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_BFLOAT16; + case EVectorType::Float8E4M3: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E4M3; + case EVectorType::Float8E5M2: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E5M2; case EVectorType::Uint8: return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8; case EVectorType::Int8: diff --git a/tests/unit/client/table/table_ut.cpp b/tests/unit/client/table/table_ut.cpp index a915f8e95d4..98ce1a65829 100644 --- a/tests/unit/client/table/table_ut.cpp +++ b/tests/unit/client/table/table_ut.cpp @@ -20,6 +20,7 @@ #include #include #include +#include using namespace NYdb; @@ -1031,3 +1032,31 @@ TEST(TtlTierSettings, ObjectKeyPrefixRoundTrip) { TTtlEvictToExternalStorageAction("/Root/eds").SerializeTo(proto); EXPECT_FALSE(proto.has_object_key_prefix()); } + +TEST(VectorIndexSettings, Float8RoundTrip) { + using TSettings = NYdb::NTable::TVectorIndexSettings; + for (const auto& [type, protoType, name] : { + std::tuple{TSettings::EVectorType::Float8E4M3, Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E4M3, "float8_e4m3"}, + std::tuple{TSettings::EVectorType::Float8E5M2, Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E5M2, "float8_e5m2"}}) { + const TSettings original{TSettings::EMetric::Euclidean, type, 1536}; + Ydb::Table::VectorIndexSettings proto; + original.SerializeTo(proto); + EXPECT_EQ(proto.vector_type(), protoType); + EXPECT_EQ(proto.vector_dimension(), 1536); + EXPECT_EQ(proto.metric(), Ydb::Table::VectorIndexSettings::DISTANCE_EUCLIDEAN); + const auto restored = TSettings::FromProto(proto); + EXPECT_EQ(restored.VectorType, type); + EXPECT_EQ(restored.VectorDimension, original.VectorDimension); + EXPECT_EQ(restored.Metric, original.Metric); + EXPECT_EQ(ToString(type), name); + + NYdb::NTable::TKMeansTreeSettings tree{original, 8, 3}; + Ydb::Table::KMeansTreeSettings treeProto; + tree.SerializeTo(treeProto); + const auto restoredTree = NYdb::NTable::TKMeansTreeSettings::FromProto(treeProto); + EXPECT_EQ(restoredTree.Settings.VectorType, type); + EXPECT_EQ(restoredTree.Settings.VectorDimension, 1536); + EXPECT_EQ(restoredTree.Clusters, 8); + EXPECT_EQ(restoredTree.Levels, 3); + } +} From 6277da580a527972459f0798547087b6c2a66285 Mon Sep 17 00:00:00 2001 From: mregrock Date: Thu, 5 Nov 2026 13:51:03 +0000 Subject: [PATCH 04/19] Add ydb admin bridge set-database-mode and refactor some commands (#57638) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 ++ src/api/grpc/draft/ydb_bridge_v1.proto | 3 +++ src/api/protos/draft/ydb_bridge.proto | 17 +++++++++++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 0c46d84dfe5..6e7e8c369a2 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -e5e6e86d8936ed5351d5db0a05a4c19eb68eeb7b +7f447d8f8735ed3f7664c1499f7e08c1994a7087 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e0aa36209d..93e122ac1fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Added draft `TBridgeClient::SetDatabaseBridgeMode()` to atomically change a database's bridge mode. `TSetDatabaseBridgeModeSettings::RetainPileName()` selects the pile to retain when disabling bridge mode. + * Fixed PersQueue topic settings losing replication-consumer and consumer-offset-mirroring attributes when copied from DescribeTopic into CreateTopic or AlterTopic settings. * Supported Topic read sessions counting replication control messages as data offsets, which could block commits after an offset gap. diff --git a/src/api/grpc/draft/ydb_bridge_v1.proto b/src/api/grpc/draft/ydb_bridge_v1.proto index 55de04acbca..ecf7d4a6284 100644 --- a/src/api/grpc/draft/ydb_bridge_v1.proto +++ b/src/api/grpc/draft/ydb_bridge_v1.proto @@ -11,6 +11,9 @@ import "src/api/protos/draft/ydb_bridge.proto"; // Service for managing cluster in bridge mode service BridgeService { + // Change bridge mode of the specified database. + rpc SetDatabaseBridgeMode(Ydb.Bridge.SetDatabaseBridgeModeRequest) returns (Ydb.Bridge.SetDatabaseBridgeModeResponse); + // Get current cluster state // Useful for monitoring and decision-making before state changes rpc GetClusterState(Ydb.Bridge.GetClusterStateRequest) returns (Ydb.Bridge.GetClusterStateResponse); diff --git a/src/api/protos/draft/ydb_bridge.proto b/src/api/protos/draft/ydb_bridge.proto index 4e3d73729ac..8cfa5f3fcb6 100644 --- a/src/api/protos/draft/ydb_bridge.proto +++ b/src/api/protos/draft/ydb_bridge.proto @@ -12,6 +12,23 @@ import "src/api/protos/ydb_bridge_common.proto"; import "src/api/protos/ydb_operation.proto"; import "google/protobuf/timestamp.proto"; +// Atomically change bridge mode of the selected database. +// Requires administrator privileges and non-uniform storage; data synchronization and cleanup continue in the background. +message SetDatabaseBridgeModeRequest { + Ydb.Operations.OperationParams operation_params = 1; + optional bool bridge_mode = 2; // Required: true for bridged storage, false for ordinary storage. + // With bridge_mode=false, retain data copies in this pile. Empty selects copies automatically. + string retain_pile_name = 3; + // Required absolute path of the target database. + string database_path = 4; +} + +message SetDatabaseBridgeModeResponse { + Ydb.Operations.Operation operation = 1; +} + +message SetDatabaseBridgeModeResult {} + message GetClusterStateRequest { Ydb.Operations.OperationParams operation_params = 1; From 4c7768073fabcf182de4570b93ceddb3983554c4 Mon Sep 17 00:00:00 2001 From: Kuzin Roman Date: Thu, 5 Nov 2026 13:51:11 +0000 Subject: [PATCH 05/19] fix requests with /Root database (#57712) --- .github/last_commit.txt | 2 +- src/client/federated_topic/ut/basic_usage_ut.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 6e7e8c369a2..587f851100f 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -7f447d8f8735ed3f7664c1499f7e08c1994a7087 +ec6c4c7aa14d42a839dca9abfe5f0ead288f1562 diff --git a/src/client/federated_topic/ut/basic_usage_ut.cpp b/src/client/federated_topic/ut/basic_usage_ut.cpp index 537c11aecc6..57e3fe28c3b 100644 --- a/src/client/federated_topic/ut/basic_usage_ut.cpp +++ b/src/client/federated_topic/ut/basic_usage_ut.cpp @@ -689,7 +689,8 @@ Y_UNIT_TEST_SUITE(BasicUsage) { for (size_t i = 0u; i < messages.size(); ++i) { auto& message = messages[i]; UNIT_ASSERT(message.GetFederatedPartitionSession()->GetReadSourceDatabaseName() == "dc1"); - UNIT_ASSERT(message.GetFederatedPartitionSession()->GetTopicPath() == setup->GetTestTopic()); + // With /Root, the server returns account/topic; this test topic has an empty account. + UNIT_ASSERT_VALUES_EQUAL(message.GetFederatedPartitionSession()->GetTopicPath(), "/" + setup->GetTestTopic()); UNIT_ASSERT(message.GetData().ends_with(message.GetFederatedPartitionSession()->GetTopicOriginDatabaseName())); UNIT_ASSERT(!sentSet.empty()); From 83e340b4326a54b922faf8ec6fe29c6b1ee4e4b7 Mon Sep 17 00:00:00 2001 From: Sergey M Date: Thu, 5 Nov 2026 13:51:20 +0000 Subject: [PATCH 06/19] Add packed uint4 and int4 vector support (#57603) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/table/table.h | 4 ++++ src/api/protos/ydb_table.proto | 2 ++ src/client/table/out.cpp | 4 ++++ src/client/table/table.cpp | 8 +++++++ tests/unit/client/table/table_ut.cpp | 29 ++++++++++++++++++++++++ 6 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 587f851100f..9b58fb5e01b 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -ec6c4c7aa14d42a839dca9abfe5f0ead288f1562 +c606ed5b41cd0050667f4a0dd907de4d54ee69fb diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index c5584c3371b..321a7e1726d 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -343,6 +343,8 @@ struct TVectorIndexSettings { BFloat16, Float8E4M3, // E4M3FN Float8E5M2, + Uint4 = 9, // Packed unsigned 4-bit coordinates [0, 15]. + Int4 = 10, // Packed signed 4-bit coordinates [-8, 7]. }; EMetric Metric = EMetric::Unspecified; @@ -377,6 +379,8 @@ struct TKMeansTreeSettings { BFloat16, Float8E4M3, // E4M3FN Float8E5M2, + Uint4 = 9, // Packed unsigned 4-bit coordinates [0, 15]. + Int4 = 10, // Packed signed 4-bit coordinates [-8, 7]. }; TVectorIndexSettings Settings; diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 36bcc266dfe..a705b884240 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -84,6 +84,8 @@ message VectorIndexSettings { VECTOR_TYPE_BFLOAT16 = 6; VECTOR_TYPE_FLOAT8_E4M3 = 7; // E4M3FN, no scale factor. VECTOR_TYPE_FLOAT8_E5M2 = 8; // E5M2, no scale factor. + VECTOR_TYPE_UINT4 = 9; // Packed unsigned 4-bit coordinates, range [0, 15]. + VECTOR_TYPE_INT4 = 10; // Packed signed 4-bit two's-complement coordinates, range [-8, 7]. VECTOR_TYPE_UINT8 = 2; VECTOR_TYPE_INT8 = 3; VECTOR_TYPE_BIT = 4; diff --git a/src/client/table/out.cpp b/src/client/table/out.cpp index 5a7bb48bdca..7f42807414a 100644 --- a/src/client/table/out.cpp +++ b/src/client/table/out.cpp @@ -62,6 +62,10 @@ Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings::EVectorType, stream, va return "float8_e4m3"; case NYdb::NTable::TVectorIndexSettings::EVectorType::Float8E5M2: return "float8_e5m2"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Uint4: + return "uint4"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Int4: + return "int4"; case NYdb::NTable::TVectorIndexSettings::EVectorType::Uint8: return "uint8"; case NYdb::NTable::TVectorIndexSettings::EVectorType::Int8: diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index dfce425e915..559f8688096 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -2906,6 +2906,10 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const Ydb::Table::VectorInd return EVectorType::Float8E4M3; case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E5M2: return EVectorType::Float8E5M2; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT4: + return EVectorType::Uint4; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT4: + return EVectorType::Int4; case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8: return EVectorType::Uint8; case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT8: @@ -2954,6 +2958,10 @@ void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E4M3; case EVectorType::Float8E5M2: return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT8_E5M2; + case EVectorType::Uint4: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT4; + case EVectorType::Int4: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT4; case EVectorType::Uint8: return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8; case EVectorType::Int8: diff --git a/tests/unit/client/table/table_ut.cpp b/tests/unit/client/table/table_ut.cpp index 98ce1a65829..1baebc4d565 100644 --- a/tests/unit/client/table/table_ut.cpp +++ b/tests/unit/client/table/table_ut.cpp @@ -1033,6 +1033,35 @@ TEST(TtlTierSettings, ObjectKeyPrefixRoundTrip) { EXPECT_FALSE(proto.has_object_key_prefix()); } +TEST(VectorIndexSettings, ReducedPrecisionRoundTrip) { + using TSettings = NYdb::NTable::TVectorIndexSettings; + for (const auto& [type, protoType, name] : { + std::tuple{TSettings::EVectorType::Uint4, Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT4, "uint4"}, + std::tuple{TSettings::EVectorType::Int4, Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT4, "int4"}}) { + const TSettings original{TSettings::EMetric::Euclidean, type, 1536}; + Ydb::Table::VectorIndexSettings proto; + original.SerializeTo(proto); + EXPECT_EQ(proto.vector_type(), protoType); + EXPECT_EQ(proto.vector_dimension(), 1536); + EXPECT_EQ(proto.metric(), Ydb::Table::VectorIndexSettings::DISTANCE_EUCLIDEAN); + const auto restored = TSettings::FromProto(proto); + EXPECT_EQ(restored.VectorType, type); + EXPECT_EQ(restored.VectorDimension, original.VectorDimension); + EXPECT_EQ(restored.Metric, original.Metric); + EXPECT_EQ(ToString(type), name); + + NYdb::NTable::TKMeansTreeSettings tree{original, 8, 3}; + Ydb::Table::KMeansTreeSettings treeProto; + tree.SerializeTo(treeProto); + const auto restoredTree = NYdb::NTable::TKMeansTreeSettings::FromProto(treeProto); + EXPECT_EQ(restoredTree.Settings.VectorType, type); + EXPECT_EQ(restoredTree.Settings.VectorDimension, 1536); + EXPECT_EQ(restoredTree.Clusters, 8); + EXPECT_EQ(restoredTree.Levels, 3); + } +} + + TEST(VectorIndexSettings, Float8RoundTrip) { using TSettings = NYdb::NTable::TVectorIndexSettings; for (const auto& [type, protoType, name] : { From 6c3cc0c2bc02516991fe5e769bb4b7e52642bd16 Mon Sep 17 00:00:00 2001 From: Evgeniy Ivanov Date: Thu, 5 Nov 2026 13:51:28 +0000 Subject: [PATCH 07/19] Add expected_generation compare-and-set to UpdateClusterState (#57882) --- .github/last_commit.txt | 2 +- src/api/protos/draft/ydb_bridge.proto | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 9b58fb5e01b..8709f14e1fb 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -c606ed5b41cd0050667f4a0dd907de4d54ee69fb +24afd18081c77c07e81e95f40a2d14af78dfd1e1 diff --git a/src/api/protos/draft/ydb_bridge.proto b/src/api/protos/draft/ydb_bridge.proto index 8cfa5f3fcb6..490af48baec 100644 --- a/src/api/protos/draft/ydb_bridge.proto +++ b/src/api/protos/draft/ydb_bridge.proto @@ -108,6 +108,18 @@ message UpdateClusterStateRequest { // configuration quorum in the new cluster state, that is, every pile not DISCONNECTED, SUSPENDED or INACTIVE // there; to go on without an unavailable pile, set it DISCONNECTED in the same update. repeated string quorum_piles = 3; + // Compare-and-set guard: the generation of the cluster state (GetClusterStateResult.generation) that this + // update is based on. When set, the update is applied only if the cluster state still has this generation + // when the change executes; the committed state then has generation expected_generation + 1. + // ABORTED: the cluster state has another generation; the update was not applied. Read the state again + // before deciding the next update. + // UNAVAILABLE with a lagging-node message: the receiving node has not yet received this generation; the + // update was not applied and may be sent again, preferably to another node. + // Other failures, such as INTERNAL_ERROR, do not tell whether the update was applied. Once a generation + // greater than expected_generation is observed, the update can no longer be applied. + // When unset, the update is applied on top of the receiving node's current cluster state. + // Nodes of older versions ignore this field. + optional uint64 expected_generation = 4; } message UpdateClusterStateResponse { From 541c5177495cadee21175091682774ffcdac9c39 Mon Sep 17 00:00:00 2001 From: risenberg <14929390+risenberg@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:51:37 +0000 Subject: [PATCH 08/19] [KIKIMR-27454] Detect lost messages for async replication on startup if all were lost and no per-message checks will be triggered on consumer. (#57837) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/topic/read_events.h | 15 +++++++++++++++ src/api/protos/ydb_topic.proto | 12 +++++++++--- src/client/topic/impl/read_session_event.cpp | 10 +++++++++- src/client/topic/impl/read_session_impl.ipp | 8 +++++++- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 8709f14e1fb..3a7345c1b61 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -24afd18081c77c07e81e95f40a2d14af78dfd1e1 +ca95a87422f70df9fcffa336cd3d8699c24a6a8c diff --git a/include/ydb-cpp-sdk/client/topic/read_events.h b/include/ydb-cpp-sdk/client/topic/read_events.h index 4099fb010be..4c00a53af53 100644 --- a/include/ydb-cpp-sdk/client/topic/read_events.h +++ b/include/ydb-cpp-sdk/client/topic/read_events.h @@ -6,6 +6,7 @@ #include +#include #include namespace NYdb::inline V3::NTopic { @@ -283,12 +284,24 @@ struct TReadSessionEvent { struct TStartPartitionSessionEvent: public TPartitionSessionAccessor, public TPrintable { explicit TStartPartitionSessionEvent(TPartitionSession::TPtr, uint64_t committedOffset, uint64_t endOffset); + explicit TStartPartitionSessionEvent(TPartitionSession::TPtr, uint64_t committedOffset, uint64_t readOffset, + uint64_t endOffset, std::optional retainedStartOffset); //! Current committed offset in partition session. uint64_t GetCommittedOffset() const { return CommittedOffset; } + //! Read position reported when the partition session starts. + uint64_t GetReadOffset() const { + return ReadOffset; + } + + //! Earliest retained offset, when reported by the server. + std::optional GetRetainedStartOffset() const { + return RetainedStartOffset; + } + //! Offset of first not existing message in partition session. uint64_t GetEndOffset() const { return EndOffset; @@ -301,7 +314,9 @@ struct TReadSessionEvent { private: uint64_t CommittedOffset; + uint64_t ReadOffset; uint64_t EndOffset; + std::optional RetainedStartOffset; }; //! Server command for stopping and destroying partition session. diff --git a/src/api/protos/ydb_topic.proto b/src/api/protos/ydb_topic.proto index d72dd262ed0..c35b785b91b 100644 --- a/src/api/protos/ydb_topic.proto +++ b/src/api/protos/ydb_topic.proto @@ -335,7 +335,7 @@ message StreamReadMessage { // Indicates that the SDK supports auto partitioning. bool auto_partitioning_support = 5; // Max in flight bytes per partition - uint64 partition_max_in_flight_bytes = 6; + uint64 partition_max_in_flight_bytes = 6; message TopicReadSettings { // Topic path. @@ -478,7 +478,9 @@ message StreamReadMessage { // Identifier of partition session whose status was requested. int64 partition_session_id = 1; - // Partition contains messages with offsets in range [start, end). + + // start is the committed offset; end is the partition end offset. + // start may precede the earliest retained offset. OffsetsRange partition_offsets = 2; // Each offset up to and including (committed_offset - 1) was fully processed. @@ -500,11 +502,15 @@ message StreamReadMessage { // Each offset up to and including (committed_offset - 1) was fully processed. int64 committed_offset = 2; - // Partition contains messages with offsets in range [start, end). + // start is the initial read offset; end is the partition end offset. + // start may precede retained_start_offset. OffsetsRange partition_offsets = 3; // Partition location, filled only when InitRequest.direct_read is true. PartitionLocation partition_location = 4; + + // Earliest offset retained by the partition + optional int64 retained_start_offset = 5; } // Signal for server that client is ready to recive data for partition. diff --git a/src/client/topic/impl/read_session_event.cpp b/src/client/topic/impl/read_session_event.cpp index b935e5daa19..aea858b3348 100644 --- a/src/client/topic/impl/read_session_event.cpp +++ b/src/client/topic/impl/read_session_event.cpp @@ -317,9 +317,17 @@ void TPrintable::DebugString(TStringBuilder& TStartPartitionSessionEvent::TStartPartitionSessionEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset, uint64_t endOffset) + : TStartPartitionSessionEvent(std::move(partitionSession), committedOffset, committedOffset, endOffset, std::nullopt) { +} + +TStartPartitionSessionEvent::TStartPartitionSessionEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset, + uint64_t readOffset, uint64_t endOffset, + std::optional retainedStartOffset) : TPartitionSessionAccessor(std::move(partitionSession)) , CommittedOffset(committedOffset) - , EndOffset(endOffset) { + , ReadOffset(readOffset) + , EndOffset(endOffset) + , RetainedStartOffset(retainedStartOffset) { } void TStartPartitionSessionEvent::Confirm(std::optional readOffset, std::optional commitOffset, std::optional maxOffset) { diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index c57fe8302ce..040b92f11f1 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -1692,11 +1692,17 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( NextPartitionStreamId += PartitionStreamIdStep; PartitionStreams.insert_or_assign(partitionSessionId, partitionStream); + std::optional retainedStartOffset; + if (msg.has_retained_start_offset()) { + retainedStartOffset = msg.retained_start_offset(); + } + // Send event to user. bool pushRes = EventsQueue->PushEvent( partitionStream, TReadSessionEvent::TStartPartitionSessionEvent( - partitionStream, msg.committed_offset(), msg.partition_offsets().end()), + partitionStream, msg.committed_offset(), msg.partition_offsets().start(), + msg.partition_offsets().end(), retainedStartOffset), deferred); if (!pushRes) { From dc580cb4ffdf4495ae928ac384e9d20282a502ef Mon Sep 17 00:00:00 2001 From: Evgeniy Ivanov Date: Thu, 5 Nov 2026 13:51:45 +0000 Subject: [PATCH 09/19] Expose expected_generation in the C++ SDK and the bridge CLI (#57950) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 3a7345c1b61..bac1f0c5f67 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -ca95a87422f70df9fcffa336cd3d8699c24a6a8c +af7239f4371ea662c57d143529d1d732539eaf79 diff --git a/CHANGELOG.md b/CHANGELOG.md index 93e122ac1fe..a5ae6bfd6a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Added `TUpdateClusterStateSettings::ExpectedGeneration()` to apply a bridge cluster state update only if the cluster state still has the given generation. + * Added draft `TBridgeClient::SetDatabaseBridgeMode()` to atomically change a database's bridge mode. `TSetDatabaseBridgeModeSettings::RetainPileName()` selects the pile to retain when disabling bridge mode. * Fixed PersQueue topic settings losing replication-consumer and consumer-offset-mirroring attributes when copied from DescribeTopic into CreateTopic or AlterTopic settings. From 628794c4a2b38656abbd41d554785b1b141f1209 Mon Sep 17 00:00:00 2001 From: YDBot Date: Thu, 5 Nov 2026 13:52:03 +0000 Subject: [PATCH 10/19] Merge pull request #57940 from ydb-platform/robots/branch-sync/merge-rightlib-261103-0045 --- .github/last_commit.txt | 2 +- util/generic/bitmap.h | 480 ++++++++++++++++--------------------- util/generic/bitmap_ut.cpp | 93 +++++++ util/str_stl.h | 2 +- 4 files changed, 305 insertions(+), 272 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index bac1f0c5f67..d6e73becde9 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -af7239f4371ea662c57d143529d1d732539eaf79 +c989daddb9374148beb8f9b21797747a2ef66d85 diff --git a/util/generic/bitmap.h b/util/generic/bitmap.h index e687cdf7519..543131a9573 100644 --- a/util/generic/bitmap.h +++ b/util/generic/bitmap.h @@ -3,8 +3,8 @@ #include "fwd.h" #include "ptr.h" #include "bitops.h" -#include "typetraits.h" -#include "algorithm.h" +#include "constant_evaluation.h" +#include "mem_copy.h" #include "utility.h" #include @@ -12,9 +12,11 @@ #include #include +#include + namespace NBitMapPrivate { // Returns the number of bits set; result is in the most significant byte - inline ui64 ByteSums(ui64 x) { + constexpr ui64 ByteSums(ui64 x) { ui64 byteSums = x - ((x & 0xAAAAAAAAAAAAAAAAULL) >> 1); byteSums = (byteSums & 0x3333333333333333ULL) + ((byteSums >> 2) & 0x3333333333333333ULL); @@ -25,129 +27,69 @@ namespace NBitMapPrivate { // better than intrinsics without -mpopcnt template - static unsigned CountBitsPrivate(T v) noexcept { + static constexpr unsigned CountBitsPrivate(T v) noexcept { return static_cast(ByteSums(v) >> 56); } - template - struct TSanitizeMask { - static constexpr TChunkType Value = ~((~TChunkType(0)) << ExtraBits); - }; - - template - struct TSanitizeMask { - static constexpr TChunkType Value = (TChunkType)~TChunkType(0u); - }; - template - struct TBigToSmallDataCopier { - static_assert(sizeof(TTargetChunk) < sizeof(TSourceChunk), "expect sizeof(TTargetChunk) < sizeof(TSourceChunk)"); - static_assert(0 == sizeof(TSourceChunk) % sizeof(TTargetChunk), "expect 0 == sizeof(TSourceChunk) % sizeof(TTargetChunk)"); - - static constexpr size_t BLOCK_SIZE = sizeof(TSourceChunk) / sizeof(TTargetChunk); - - union TCnv { - TSourceChunk BigData; - TTargetChunk SmallData[BLOCK_SIZE]; - }; - - static inline void CopyChunk(TTargetChunk* target, TSourceChunk source) { - TCnv c; - c.BigData = source; -#if defined(_big_endian_) - ::ReverseCopy(c.SmallData, c.SmallData + Y_ARRAY_SIZE(c.SmallData), target); -#else - ::Copy(c.SmallData, c.SmallData + Y_ARRAY_SIZE(c.SmallData), target); -#endif - } - - static inline void Copy(TTargetChunk* target, size_t targetSize, const TSourceChunk* source, size_t sourceSize) { - Y_ASSERT(targetSize >= sourceSize * BLOCK_SIZE); - if (targetSize > sourceSize * BLOCK_SIZE) { - ::Fill(target + sourceSize * BLOCK_SIZE, target + targetSize, 0); - } - for (size_t i = 0; i < sourceSize; ++i) { - CopyChunk(target + i * BLOCK_SIZE, source[i]); + Y_FORCE_INLINE constexpr void CopyData(TTargetChunk* target, size_t targetSize, const TSourceChunk* source, size_t sourceSize) { + Y_ASSERT(targetSize * sizeof(TTargetChunk) >= sourceSize * sizeof(TSourceChunk)); +#if defined(_little_endian_) + if constexpr (sizeof(TTargetChunk) != sizeof(TSourceChunk)) { + if (!IsConstantEvaluated()) { + // Little-endian chunks have the same byte layout regardless of chunk width. + // Keep the runtime copy vectorizable; constant evaluation uses shifts below. + const size_t sourceBytes = sourceSize * sizeof(TSourceChunk); + const size_t targetBytes = targetSize * sizeof(TTargetChunk); + auto* targetData = reinterpret_cast(target); + MemMove(targetData, reinterpret_cast(source), sourceBytes); + if (sourceBytes < targetBytes) { + ::memset(targetData + sourceBytes, 0, targetBytes - sourceBytes); + } + return; } } - }; - - template - struct TSmallToBigDataCopier { - static_assert(sizeof(TTargetChunk) > sizeof(TSourceChunk), "expect sizeof(TTargetChunk) > sizeof(TSourceChunk)"); - static_assert(0 == sizeof(TTargetChunk) % sizeof(TSourceChunk), "expect 0 == sizeof(TTargetChunk) % sizeof(TSourceChunk)"); - - static constexpr size_t BLOCK_SIZE = sizeof(TTargetChunk) / sizeof(TSourceChunk); - - union TCnv { - TSourceChunk SmallData[BLOCK_SIZE]; - TTargetChunk BigData; - }; - - static inline TTargetChunk CopyFullChunk(const TSourceChunk* source) { - TCnv c; -#if defined(_big_endian_) - ::ReverseCopy(source, source + BLOCK_SIZE, c.SmallData); -#else - ::Copy(source, source + BLOCK_SIZE, c.SmallData); -#endif - return c.BigData; - } - - static inline TTargetChunk CopyPartChunk(const TSourceChunk* source, size_t count) { - Y_ASSERT(count <= BLOCK_SIZE); - TCnv c; - c.BigData = 0; -#if defined(_big_endian_) - ::ReverseCopy(source, source + count, c.SmallData); -#else - ::Copy(source, source + count, c.SmallData); #endif - return c.BigData; - } - - static inline void Copy(TTargetChunk* target, size_t targetSize, const TSourceChunk* source, size_t sourceSize) { - Y_ASSERT(targetSize * BLOCK_SIZE >= sourceSize); - if (targetSize * BLOCK_SIZE > sourceSize) { - ::Fill(target + sourceSize / BLOCK_SIZE, target + targetSize, 0); - } - size_t i = 0; - for (; i < sourceSize / BLOCK_SIZE; ++i) { - target[i] = CopyFullChunk(source + i * BLOCK_SIZE); - } - if (0 != sourceSize % BLOCK_SIZE) { - target[i] = CopyPartChunk(source + i * BLOCK_SIZE, sourceSize % BLOCK_SIZE); - } - } - }; - - template - struct TUniformDataCopier { - static inline void Copy(TChunk* target, size_t targetSize, const TChunk* source, size_t sourceSize) { - Y_ASSERT(targetSize >= sourceSize); + if constexpr (sizeof(TTargetChunk) == sizeof(TSourceChunk)) { for (size_t i = 0; i < sourceSize; ++i) { target[i] = source[i]; } for (size_t i = sourceSize; i < targetSize; ++i) { target[i] = 0; } + } else if constexpr (sizeof(TTargetChunk) < sizeof(TSourceChunk)) { + constexpr size_t blockSize = sizeof(TSourceChunk) / sizeof(TTargetChunk); + for (size_t i = 0; i < sourceSize; ++i) { + for (size_t j = 0; j < blockSize; ++j) { + target[i * blockSize + j] = static_cast(source[i] >> (j * 8 * sizeof(TTargetChunk))); + } + } + for (size_t i = sourceSize * blockSize; i < targetSize; ++i) { + target[i] = 0; + } + } else { + constexpr size_t blockSize = sizeof(TTargetChunk) / sizeof(TSourceChunk); + for (size_t i = 0; i < targetSize; ++i) { + TTargetChunk chunk = 0; + for (size_t j = 0; j < blockSize && i * blockSize + j < sourceSize; ++j) { + chunk |= static_cast(source[i * blockSize + j]) << (j * 8 * sizeof(TSourceChunk)); + } + target[i] = chunk; + } } - }; - - template - struct TIsSmaller { - enum { - Result = sizeof(TFirst) < sizeof(TSecond) - }; - }; - - template - struct TDataCopier: public std::conditional_t::value, TUniformDataCopier, std::conditional_t::Result, TBigToSmallDataCopier, TSmallToBigDataCopier>> { - }; + } - template - inline void CopyData(TTargetChunk* target, size_t targetSize, const TSourceChunk* source, size_t sourceSize) { - TDataCopier::Copy(target, targetSize, source, sourceSize); + template + constexpr unsigned CountTrailingZeroBits(T value) noexcept { + if (!IsConstantEvaluated()) { + return ::CountTrailingZeroBits(value); + } + unsigned count = 0; + while (!(value & 1)) { + value >>= 1; + ++count; + } + return count; } template @@ -156,27 +98,27 @@ namespace NBitMapPrivate { static constexpr size_t Size = (BitCount + 8 * sizeof(TChunk) - 1) / (8 * sizeof(TChunk)); - TChunk Data[Size]; + TChunk Data[Size]{}; - TFixedStorage() { - Zero(Data); - } - - TFixedStorage(const TFixedStorage& st) { + constexpr TFixedStorage() = default; + // An element-wise copy lets the optimizer combine copying with subsequent bit operations. + constexpr TFixedStorage(const TFixedStorage& st) { for (size_t i = 0; i < Size; ++i) { Data[i] = st.Data[i]; } } template - TFixedStorage(const TOtherChunk* data, size_t size) { + Y_FORCE_INLINE constexpr TFixedStorage(const TOtherChunk* data, size_t size) { Y_ABORT_UNLESS(Size * sizeof(TChunk) >= size * sizeof(TOtherChunk), "Exceeding bitmap storage capacity"); CopyData(Data, Size, data, size); } - Y_FORCE_INLINE void Swap(TFixedStorage& st) { + Y_FORCE_INLINE constexpr void Swap(TFixedStorage& st) { for (size_t i = 0; i < Size; ++i) { - DoSwap(Data[i], st.Data[i]); + const TChunk tmp = Data[i]; + Data[i] = st.Data[i]; + st.Data[i] = tmp; } } @@ -193,8 +135,11 @@ namespace NBitMapPrivate { return bitSize <= BitCount; } - Y_FORCE_INLINE void Sanitize() { - Data[Size - 1] &= TSanitizeMask::Value; + Y_FORCE_INLINE constexpr void Sanitize() { + constexpr size_t extraBits = BitCount % (8 * sizeof(TChunk)); + if constexpr (extraBits != 0) { + Data[Size - 1] &= (TChunk{1} << extraBits) - 1; + } } }; @@ -284,16 +229,6 @@ namespace NBitMapPrivate { } }; - template - struct TDivCount { - static constexpr size_t Value = 1 + TDivCount<(num >> 1)>::Value; - }; - - template <> - struct TDivCount<0> { - static constexpr size_t Value = 0; - }; - } // namespace NBitMapPrivate template @@ -315,12 +250,12 @@ class TBitMapOps { using TThis = TBitMapOps; private: - static_assert(std::is_unsigned::value, "expect std::is_unsigned::value"); + static_assert(std::is_unsigned_v, "expect std::is_unsigned_v"); static constexpr size_t BitsPerChunk = 8 * sizeof(TChunk); static constexpr TChunk ModMask = static_cast(BitsPerChunk - 1); - static constexpr size_t DivCount = NBitMapPrivate::TDivCount::Value - 1; - static constexpr TChunk FullChunk = (TChunk)~TChunk(0); + static constexpr size_t DivCount = MostSignificantBitCT(BitsPerChunk); + static constexpr TChunk FullChunk = static_cast(~TChunk{0}); template friend class TBitMapOps; @@ -340,7 +275,7 @@ class TBitMapOps { TChunk* Chunk; size_t Offset; - TReference(TChunk* c, size_t offset) + constexpr TReference(TChunk* c, size_t offset) : Chunk(c) , Offset(offset) { @@ -349,7 +284,7 @@ class TBitMapOps { public: ~TReference() = default; - Y_FORCE_INLINE TReference& operator=(bool val) { + Y_FORCE_INLINE constexpr TReference& operator=(bool val) { if (val) { *Chunk |= static_cast(1) << Offset; } else { @@ -359,25 +294,19 @@ class TBitMapOps { return *this; } - Y_FORCE_INLINE TReference& operator=(const TReference& ref) { - if (ref) { - *Chunk |= static_cast(1) << Offset; - } else { - *Chunk &= ~(static_cast(1) << Offset); - } - - return *this; + Y_FORCE_INLINE constexpr TReference& operator=(const TReference& ref) { + return *this = static_cast(ref); } - Y_FORCE_INLINE bool operator~() const { + Y_FORCE_INLINE constexpr bool operator~() const { return 0 == (*Chunk & (static_cast(1) << Offset)); } - Y_FORCE_INLINE operator bool() const { + Y_FORCE_INLINE constexpr operator bool() const { return 0 != (*Chunk & (static_cast(1) << Offset)); } - Y_FORCE_INLINE TReference& Flip() { + Y_FORCE_INLINE constexpr TReference& Flip() { *Chunk ^= static_cast(1) << Offset; return *this; } @@ -397,7 +326,7 @@ class TBitMapOps { }; template - void UpdateRange(size_t start, size_t end) { + constexpr void UpdateRange(size_t start, size_t end) { const size_t startChunk = start >> DivCount; const size_t startBitOffset = start & ModMask; @@ -419,28 +348,28 @@ class TBitMapOps { } public: - TBitMapOps() = default; + constexpr TBitMapOps() = default; - TBitMapOps(TChunk val) { + constexpr TBitMapOps(TChunk val) { Mask.Data[0] = val; Mask.Sanitize(); } - TBitMapOps(const TThis&) = default; + constexpr TBitMapOps(const TThis&) = default; template - TBitMapOps(const TBitMapOps& bitmap) + constexpr TBitMapOps(const TBitMapOps& bitmap) : Mask(bitmap.Mask.Data, bitmap.Mask.GetChunkCapacity()) { Mask.Sanitize(); } template - Y_FORCE_INLINE bool operator==(const TBitMapOps& bitmap) const { + Y_FORCE_INLINE constexpr bool operator==(const TBitMapOps& bitmap) const { return Equal(bitmap); } - Y_FORCE_INLINE TThis& operator=(const TThis& bitmap) { + Y_FORCE_INLINE constexpr TThis& operator=(const TThis& bitmap) { if (this != &bitmap) { TThis bm(bitmap); Swap(bm); @@ -449,79 +378,79 @@ class TBitMapOps { } template - Y_FORCE_INLINE TThis& operator=(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& operator=(const TBitMapOps& bitmap) { TThis bm(bitmap); Swap(bm); return *this; } template - Y_FORCE_INLINE TThis& operator&=(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& operator&=(const TBitMapOps& bitmap) { return And(bitmap); } - Y_FORCE_INLINE TThis& operator&=(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& operator&=(const TChunk& val) { return And(val); } template - Y_FORCE_INLINE TThis& operator|=(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& operator|=(const TBitMapOps& bitmap) { return Or(bitmap); } - Y_FORCE_INLINE TThis& operator|=(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& operator|=(const TChunk& val) { return Or(val); } template - Y_FORCE_INLINE TThis& operator^=(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& operator^=(const TBitMapOps& bitmap) { return Xor(bitmap); } - Y_FORCE_INLINE TThis& operator^=(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& operator^=(const TChunk& val) { return Xor(val); } template - Y_FORCE_INLINE TThis& operator-=(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& operator-=(const TBitMapOps& bitmap) { return SetDifference(bitmap); } - Y_FORCE_INLINE TThis& operator-=(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& operator-=(const TChunk& val) { return SetDifference(val); } - Y_FORCE_INLINE TThis& operator<<=(size_t pos) { + Y_FORCE_INLINE constexpr TThis& operator<<=(size_t pos) { return LShift(pos); } - Y_FORCE_INLINE TThis& operator>>=(size_t pos) { + Y_FORCE_INLINE constexpr TThis& operator>>=(size_t pos) { return RShift(pos); } - Y_FORCE_INLINE TThis operator<<(size_t pos) const { + Y_FORCE_INLINE constexpr TThis operator<<(size_t pos) const { return TThis(*this).LShift(pos); } - Y_FORCE_INLINE TThis operator>>(size_t pos) const { + Y_FORCE_INLINE constexpr TThis operator>>(size_t pos) const { return TThis(*this).RShift(pos); } - Y_FORCE_INLINE bool operator[](size_t pos) const { + Y_FORCE_INLINE constexpr bool operator[](size_t pos) const { return Get(pos); } - Y_FORCE_INLINE TReference operator[](size_t pos) { + Y_FORCE_INLINE constexpr TReference operator[](size_t pos) { const bool fitStorage = Mask.ExpandBitSize(pos + 1); Y_ASSERT(fitStorage); return TReference(&Mask.Data[pos >> DivCount], ModMask & pos); } - Y_FORCE_INLINE void Swap(TThis& bitmap) { - DoSwap(Mask, bitmap.Mask); + Y_FORCE_INLINE constexpr void Swap(TThis& bitmap) { + Mask.Swap(bitmap.Mask); } - Y_FORCE_INLINE TThis& Set(size_t pos) { + Y_FORCE_INLINE constexpr TThis& Set(size_t pos) { const bool fitStorage = Mask.ExpandBitSize(pos + 1); Y_ASSERT(fitStorage); Mask.Data[pos >> DivCount] |= static_cast(1) << (pos & ModMask); @@ -529,7 +458,7 @@ class TBitMapOps { } // Fills the specified [start, end) bit range by the 1. Other bits are kept unchanged - TThis& Set(size_t start, size_t end) { + constexpr TThis& Set(size_t start, size_t end) { Y_ASSERT(start <= end); if (start < end) { Reserve(end); @@ -538,7 +467,7 @@ class TBitMapOps { return *this; } - Y_FORCE_INLINE TThis& Reset(size_t pos) { + Y_FORCE_INLINE constexpr TThis& Reset(size_t pos) { if ((pos >> DivCount) < Mask.GetChunkCapacity()) { Mask.Data[pos >> DivCount] &= ~(static_cast(1) << (pos & ModMask)); } @@ -546,7 +475,7 @@ class TBitMapOps { } // Clears the specified [start, end) bit range. Other bits are kept unchanged - TThis& Reset(size_t start, size_t end) { + constexpr TThis& Reset(size_t start, size_t end) { Y_ASSERT(start <= end); if (start < end && (start >> DivCount) < Mask.GetChunkCapacity()) { UpdateRange(start, Min(end, Mask.GetBitCapacity())); @@ -554,14 +483,14 @@ class TBitMapOps { return *this; } - Y_FORCE_INLINE TThis& Flip(size_t pos) { + Y_FORCE_INLINE constexpr TThis& Flip(size_t pos) { const bool fitStorage = Mask.ExpandBitSize(pos + 1); Y_ASSERT(fitStorage); Mask.Data[pos >> DivCount] ^= static_cast(1) << (pos & ModMask); return *this; } - Y_FORCE_INLINE bool Get(size_t pos) const { + Y_FORCE_INLINE constexpr bool Get(size_t pos) const { if ((pos >> DivCount) < Mask.GetChunkCapacity()) { return Mask.Data[pos >> DivCount] & (static_cast(1) << (pos & ModMask)); } @@ -569,46 +498,36 @@ class TBitMapOps { } template - void Export(size_t pos, TTo& to) const { - static_assert(std::is_unsigned::value, "expect std::is_unsigned::value"); + constexpr void Export(size_t pos, TTo& to) const { + static_assert(std::is_unsigned_v, "expect std::is_unsigned_v"); + using TExport = std::common_type_t; to = 0; - size_t chunkpos = pos >> DivCount; - if (chunkpos >= Mask.GetChunkCapacity()) { - return; - } - if ((pos & ModMask) == 0) { - if (sizeof(TChunk) >= sizeof(TTo)) { - to = (TTo)Mask.Data[chunkpos]; - } else { // if (sizeof(TChunk) < sizeof(TTo)) - NBitMapPrivate::CopyData(&to, 1, Mask.Data + chunkpos, Min(((sizeof(TTo) * 8) >> DivCount), Mask.GetChunkCapacity() - chunkpos)); - } - } else if ((pos & (sizeof(TTo) * 8 - 1)) == 0 && sizeof(TChunk) >= 2 * sizeof(TTo)) { - to = (TTo)(Mask.Data[chunkpos] >> (pos & ModMask)); - } else { - static constexpr size_t copyToSize = (sizeof(TChunk) >= sizeof(TTo)) ? (sizeof(TChunk) / sizeof(TTo)) + 2 : 3; - TTo temp[copyToSize] = {0, 0}; - // or use non defined by now TBitmap::CopyData,RShift(pos & ModMask),Export(0,to) - NBitMapPrivate::CopyData(temp, copyToSize, Mask.Data + chunkpos, Min((sizeof(TTo) / sizeof(TChunk)) + 1, Mask.GetChunkCapacity() - chunkpos)); - to = (temp[0] >> (pos & ModMask)) | (temp[1] << (8 * sizeof(TTo) - (pos & ModMask))); + size_t chunk = pos >> DivCount; + size_t offset = pos & ModMask; + for (size_t written = 0; written < 8 * sizeof(TTo) && chunk < Mask.GetChunkCapacity(); ++chunk) { + to |= static_cast(Mask.Data[chunk] >> offset) << written; + written += Min(BitsPerChunk - offset, 8 * sizeof(TTo) - written); + offset = 0; } } - Y_FORCE_INLINE bool Test(size_t n) const { + Y_FORCE_INLINE constexpr bool Test(size_t n) const { return Get(n); } - Y_FORCE_INLINE TThis& Push(bool val) { + Y_FORCE_INLINE constexpr TThis& Push(bool val) { LShift(1); return val ? Set(0) : *this; } - Y_FORCE_INLINE bool Pop() { + Y_FORCE_INLINE constexpr bool Pop() { bool val = Get(0); - return RShift(1), val; + RShift(1); + return val; } // Clear the entire bitmap. Current capacity is kept unchanged - Y_FORCE_INLINE TThis& Clear() { + Y_FORCE_INLINE constexpr TThis& Clear() { for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) { Mask.Data[i] = 0; } @@ -620,21 +539,26 @@ class TBitMapOps { return Mask.GetBitCapacity(); } - Y_FORCE_INLINE void Reserve(size_t bitCount) { + Y_FORCE_INLINE constexpr void Reserve(size_t bitCount) { Y_ABORT_UNLESS(Mask.ExpandBitSize(bitCount), "Exceeding bitmap storage capacity"); } - Y_FORCE_INLINE size_t ValueBitCount() const { + Y_FORCE_INLINE constexpr size_t ValueBitCount() const { size_t nonZeroChunk = Mask.GetChunkCapacity() - 1; while (nonZeroChunk != 0 && !Mask.Data[nonZeroChunk]) { --nonZeroChunk; } - return nonZeroChunk || Mask.Data[nonZeroChunk] - ? nonZeroChunk * BitsPerChunk + GetValueBitCount(TIntType(Mask.Data[nonZeroChunk])) - : 0; + const TChunk value = Mask.Data[nonZeroChunk]; + if (!value) { + return 0; + } + const size_t chunkBits = IsConstantEvaluated() + ? MostSignificantBitCT(value) + 1 + : GetValueBitCount(static_cast(value)); + return nonZeroChunk * BitsPerChunk + chunkBits; } - Y_PURE_FUNCTION Y_FORCE_INLINE bool Empty() const { + Y_PURE_FUNCTION Y_FORCE_INLINE constexpr bool Empty() const { for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) { if (Mask.Data[i]) { return false; @@ -643,7 +567,7 @@ class TBitMapOps { return true; } - bool HasAny(const TThis& bitmap) const { + constexpr bool HasAny(const TThis& bitmap) const { for (size_t i = 0; i < Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity()); ++i) { if (0 != (Mask.Data[i] & bitmap.Mask.Data[i])) { return true; @@ -653,15 +577,15 @@ class TBitMapOps { } template - Y_FORCE_INLINE bool HasAny(const TBitMapOps& bitmap) const { + Y_FORCE_INLINE constexpr bool HasAny(const TBitMapOps& bitmap) const { return HasAny(TThis(bitmap)); } - Y_FORCE_INLINE bool HasAny(const TChunk& val) const { + Y_FORCE_INLINE constexpr bool HasAny(const TChunk& val) const { return 0 != (Mask.Data[0] & val); } - bool HasAll(const TThis& bitmap) const { + constexpr bool HasAll(const TThis& bitmap) const { for (size_t i = 0; i < Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity()); ++i) { if (bitmap.Mask.Data[i] != (Mask.Data[i] & bitmap.Mask.Data[i])) { return false; @@ -676,15 +600,15 @@ class TBitMapOps { } template - Y_FORCE_INLINE bool HasAll(const TBitMapOps& bitmap) const { + Y_FORCE_INLINE constexpr bool HasAll(const TBitMapOps& bitmap) const { return HasAll(TThis(bitmap)); } - Y_FORCE_INLINE bool HasAll(const TChunk& val) const { + Y_FORCE_INLINE constexpr bool HasAll(const TChunk& val) const { return (Mask.Data[0] & val) == val; } - TThis& And(const TThis& bitmap) { + constexpr TThis& And(const TThis& bitmap) { // Don't expand capacity here, because resulting bits in positions // that are greater than the size of one of these bitmaps will be zero for (size_t i = 0; i < Min(bitmap.Mask.GetChunkCapacity(), Mask.GetChunkCapacity()); ++i) { @@ -698,11 +622,11 @@ class TBitMapOps { } template - Y_FORCE_INLINE TThis& And(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& And(const TBitMapOps& bitmap) { return And(TThis(bitmap)); } - Y_FORCE_INLINE TThis& And(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& And(const TChunk& val) { Mask.Data[0] &= val; for (size_t i = 1; i < Mask.GetChunkCapacity(); ++i) { Mask.Data[i] = 0; @@ -710,7 +634,7 @@ class TBitMapOps { return *this; } - TThis& Or(const TThis& bitmap) { + constexpr TThis& Or(const TThis& bitmap) { const size_t valueBitCount = bitmap.ValueBitCount(); if (valueBitCount) { // Memory optimization: expand size only for non-zero bits @@ -723,17 +647,17 @@ class TBitMapOps { } template - Y_FORCE_INLINE TThis& Or(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& Or(const TBitMapOps& bitmap) { return Or(TThis(bitmap)); } - Y_FORCE_INLINE TThis& Or(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& Or(const TChunk& val) { Mask.Data[0] |= val; Mask.Sanitize(); return *this; } - TThis& Xor(const TThis& bitmap) { + constexpr TThis& Xor(const TThis& bitmap) { Reserve(bitmap.Size()); for (size_t i = 0; i < bitmap.Mask.GetChunkCapacity(); ++i) { Mask.Data[i] ^= bitmap.Mask.Data[i]; @@ -742,17 +666,17 @@ class TBitMapOps { } template - Y_FORCE_INLINE TThis& Xor(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& Xor(const TBitMapOps& bitmap) { return Xor(TThis(bitmap)); } - Y_FORCE_INLINE TThis& Xor(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& Xor(const TChunk& val) { Mask.Data[0] ^= val; Mask.Sanitize(); return *this; } - TThis& SetDifference(const TThis& bitmap) { + constexpr TThis& SetDifference(const TThis& bitmap) { for (size_t i = 0; i < Min(bitmap.Mask.GetChunkCapacity(), Mask.GetChunkCapacity()); ++i) { Mask.Data[i] &= ~bitmap.Mask.Data[i]; } @@ -760,16 +684,16 @@ class TBitMapOps { } template - Y_FORCE_INLINE TThis& SetDifference(const TBitMapOps& bitmap) { + Y_FORCE_INLINE constexpr TThis& SetDifference(const TBitMapOps& bitmap) { return SetDifference(TThis(bitmap)); } - Y_FORCE_INLINE TThis& SetDifference(const TChunk& val) { + Y_FORCE_INLINE constexpr TThis& SetDifference(const TChunk& val) { Mask.Data[0] &= ~val; return *this; } - Y_FORCE_INLINE TThis& Flip() { + Y_FORCE_INLINE constexpr TThis& Flip() { for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) { Mask.Data[i] = ~Mask.Data[i]; } @@ -777,7 +701,7 @@ class TBitMapOps { return *this; } - TThis& LShift(size_t shift) { + constexpr TThis& LShift(size_t shift) { if (shift != 0) { const size_t valueBitCount = ValueBitCount(); // Do nothing for the empty bitmap @@ -790,8 +714,8 @@ class TBitMapOps { Mask.ExpandBitSize(valueBitCount + shift); if (offset == 0) { - for (size_t i = Mask.GetChunkCapacity() - 1; i >= eshift; --i) { - Mask.Data[i] = Mask.Data[i - eshift]; + for (size_t i = Mask.GetChunkCapacity(); i > eshift; --i) { + Mask.Data[i - 1] = Mask.Data[i - eshift - 1]; } } else { for (size_t i = Mask.GetChunkCapacity() - 1; i > eshift; --i) { @@ -812,7 +736,7 @@ class TBitMapOps { return *this; } - TThis& RShift(size_t shift) { + constexpr TThis& RShift(size_t shift) { if (shift != 0) { const size_t eshift = shift / BitsPerChunk; const size_t offset = shift % BitsPerChunk; @@ -845,7 +769,7 @@ class TBitMapOps { // Applies bitmap at the specified offset using OR operator. // This method is an optimized combination of Or() and LShift(), which allows reducing memory allocation // when combining long dynamic bitmaps. - TThis& Or(const TThis& bitmap, size_t offset) { + constexpr TThis& Or(const TThis& bitmap, size_t offset) { if (0 == offset) { return Or(bitmap); } @@ -878,7 +802,7 @@ class TBitMapOps { return *this; } - bool Equal(const TThis& bitmap) const { + constexpr bool Equal(const TThis& bitmap) const { if (Mask.GetChunkCapacity() > bitmap.Mask.GetChunkCapacity()) { for (size_t i = bitmap.Mask.GetChunkCapacity(); i < Mask.GetChunkCapacity(); ++i) { if (0 != Mask.Data[i]) { @@ -902,13 +826,28 @@ class TBitMapOps { } template - Y_FORCE_INLINE bool Equal(const TBitMapOps& bitmap) const { + Y_FORCE_INLINE constexpr bool Equal(const TBitMapOps& bitmap) const { return Equal(TThis(bitmap)); } - int Compare(const TThis& bitmap) const { + constexpr int Compare(const TThis& bitmap) const { size_t size = Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity()); - int res = ::memcmp(Mask.Data, bitmap.Mask.Data, size * sizeof(TChunk)); + int res = 0; + if (IsConstantEvaluated()) { + // Preserve memcmp's byte order, including on big-endian platforms. + for (size_t i = 0; i < size && res == 0; ++i) { + for (size_t byte = 0; byte < sizeof(TChunk) && res == 0; ++byte) { +#if defined(_big_endian_) + const size_t shift = 8 * (sizeof(TChunk) - byte - 1); +#else + const size_t shift = 8 * byte; +#endif + res = int(static_cast(Mask.Data[i] >> shift)) - int(static_cast(bitmap.Mask.Data[i] >> shift)); + } + } + } else { + res = ::memcmp(Mask.Data, bitmap.Mask.Data, size * sizeof(TChunk)); + } if (0 != res || Mask.GetChunkCapacity() == bitmap.Mask.GetChunkCapacity()) { return res; } @@ -930,20 +869,20 @@ class TBitMapOps { } template - Y_FORCE_INLINE int Compare(const TBitMapOps& bitmap) const { + Y_FORCE_INLINE constexpr int Compare(const TBitMapOps& bitmap) const { return Compare(TThis(bitmap)); } // For backward compatibility - Y_FORCE_INLINE static int Compare(const TThis& l, const TThis& r) { + Y_FORCE_INLINE static constexpr int Compare(const TThis& l, const TThis& r) { return l.Compare(r); } - size_t FirstNonZeroBit() const { + constexpr size_t FirstNonZeroBit() const { for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) { if (Mask.Data[i]) { // CountTrailingZeroBits() expects unsigned types not smaller than unsigned int. So, convert before calling - return BitsPerChunk * i + CountTrailingZeroBits(TIntType(Mask.Data[i])); + return BitsPerChunk * i + NBitMapPrivate::CountTrailingZeroBits(TIntType(Mask.Data[i])); } } return Size(); @@ -955,16 +894,16 @@ class TBitMapOps { // ... // } // See Y_FOR_EACH_BIT macro definition at the bottom - size_t NextNonZeroBit(size_t pos) const { + constexpr size_t NextNonZeroBit(size_t pos) const { size_t i = (pos + 1) >> DivCount; if (i < Mask.GetChunkCapacity()) { const size_t offset = (pos + 1) & ModMask; // Process the current chunk if (offset) { // Zero already iterated trailing bits using mask - const TChunk val = Mask.Data[i] & ((~TChunk(0)) << offset); + const TChunk val = Mask.Data[i] & (FullChunk << offset); if (val) { - return BitsPerChunk * i + CountTrailingZeroBits(TIntType(val)); + return BitsPerChunk * i + NBitMapPrivate::CountTrailingZeroBits(TIntType(val)); } // Continue with other chunks ++i; @@ -972,14 +911,14 @@ class TBitMapOps { for (; i < Mask.GetChunkCapacity(); ++i) { if (Mask.Data[i]) { - return BitsPerChunk * i + CountTrailingZeroBits(TIntType(Mask.Data[i])); + return BitsPerChunk * i + NBitMapPrivate::CountTrailingZeroBits(TIntType(Mask.Data[i])); } } } return Size(); } - Y_FORCE_INLINE size_t Count() const { + Y_FORCE_INLINE constexpr size_t Count() const { size_t count = 0; for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) { count += ::NBitMapPrivate::CountBitsPrivate(Mask.Data[i]); @@ -1015,7 +954,7 @@ class TBitMapOps { Mask.Sanitize(); } - inline size_t Hash() const { + constexpr size_t Hash() const { THash chunkHasher; size_t hash = chunkHasher(0); @@ -1030,7 +969,7 @@ class TBitMapOps { return hash; } - inline const TChunk* GetChunks() const { + constexpr const TChunk* GetChunks() const { return Mask.Data; } @@ -1040,67 +979,67 @@ class TBitMapOps { }; template -inline TBitMapOps operator&(const TBitMapOps& x, const TBitMapOps& y) { +constexpr TBitMapOps operator&(const TBitMapOps& x, const TBitMapOps& y) { return TBitMapOps(x).And(y); } template -inline TBitMapOps operator&(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { +constexpr TBitMapOps operator&(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { return TBitMapOps(x).And(y); } template -inline TBitMapOps operator&(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { +constexpr TBitMapOps operator&(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { return TBitMapOps(x).And(y); } template -inline TBitMapOps operator|(const TBitMapOps& x, const TBitMapOps& y) { +constexpr TBitMapOps operator|(const TBitMapOps& x, const TBitMapOps& y) { return TBitMapOps(x).Or(y); } template -inline TBitMapOps operator|(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { +constexpr TBitMapOps operator|(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { return TBitMapOps(x).Or(y); } template -inline TBitMapOps operator|(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { +constexpr TBitMapOps operator|(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { return TBitMapOps(x).Or(y); } template -inline TBitMapOps operator^(const TBitMapOps& x, const TBitMapOps& y) { +constexpr TBitMapOps operator^(const TBitMapOps& x, const TBitMapOps& y) { return TBitMapOps(x).Xor(y); } template -inline TBitMapOps operator^(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { +constexpr TBitMapOps operator^(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { return TBitMapOps(x).Xor(y); } template -inline TBitMapOps operator^(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { +constexpr TBitMapOps operator^(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { return TBitMapOps(x).Xor(y); } template -inline TBitMapOps operator-(const TBitMapOps& x, const TBitMapOps& y) { +constexpr TBitMapOps operator-(const TBitMapOps& x, const TBitMapOps& y) { return TBitMapOps(x).SetDifference(y); } template -inline TBitMapOps operator-(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { +constexpr TBitMapOps operator-(const TBitMapOps& x, const typename TBitMapOps::TChunk& y) { return TBitMapOps(x).SetDifference(y); } template -inline TBitMapOps operator-(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { +constexpr TBitMapOps operator-(const typename TBitMapOps::TChunk& x, const TBitMapOps& y) { return TBitMapOps(x).SetDifference(y); } template -inline TBitMapOps operator~(const TBitMapOps& x) { +constexpr TBitMapOps operator~(const TBitMapOps& x) { return TBitMapOps(x).Flip(); } @@ -1112,21 +1051,18 @@ class TBitMap: public TBitMapOps> { using TBase = TBitMapOps>; public: - TBitMap() - : TBase() - { - } + constexpr TBitMap() = default; - TBitMap(typename TBase::TChunk val) + constexpr TBitMap(typename TBase::TChunk val) : TBase(val) { } - TBitMap(const TBitMap&) = default; - TBitMap& operator=(const TBitMap&) = default; + constexpr TBitMap(const TBitMap&) = default; + constexpr TBitMap& operator=(const TBitMap&) = default; template - TBitMap(const TBitMapOps& bitmap) + constexpr TBitMap(const TBitMapOps& bitmap) : TBase(bitmap) { } @@ -1138,7 +1074,11 @@ using TDynBitMap = TBitMapOps>; template struct THash> { - size_t operator()(const TBitMapOps& elem) const { + constexpr size_t operator()(const TBitMapOps& elem) const { return elem.Hash(); } }; + +template +struct THash>: THash>> { +}; diff --git a/util/generic/bitmap_ut.cpp b/util/generic/bitmap_ut.cpp index e7b4ecba7b1..9b8af4695b2 100644 --- a/util/generic/bitmap_ut.cpp +++ b/util/generic/bitmap_ut.cpp @@ -37,7 +37,100 @@ } \ } +namespace { + template + constexpr bool TestConstexprConversion() { + TBitMap<137, TFrom> source; + source.Set(0).Set(7).Set(15).Set(31).Set(63).Set(64).Set(100).Set(136); + TBitMap<192, TTo> copy(source); + TBitMap<192, TTo> assigned; + assigned = source; + const TBitMap<192, TFrom> roundTrip(copy); + for (size_t bit = 0; bit < 192; ++bit) { + if (copy.Get(bit) != source.Get(bit) || assigned.Get(bit) != source.Get(bit) || roundTrip.Get(bit) != source.Get(bit)) { + return false; + } + } + return true; + } + + template + constexpr bool TestConstexprExport() { + TBitMap<137, TChunk> bits; + for (size_t bit = 0; bit < bits.Size(); ++bit) { + if (bit % 7 < 3) { + bits.Set(bit); + } + } + for (size_t pos = 0; pos < 150; ++pos) { + TExport actual = 0; + bits.Export(pos, actual); + TExport expected = 0; + for (size_t bit = 0; bit < 8 * sizeof(TExport); ++bit) { + if (bits.Get(pos + bit)) { + expected |= TExport{1} << bit; + } + } + if (actual != expected) { + return false; + } + } + return true; + } + + constexpr bool TestConstexprCompare() { + const TBitMap<128> lowByte(0xff); + const TBitMap<128> highByte(0x100); +#if defined(_big_endian_) + return lowByte.Compare(highByte) < 0 && highByte.Compare(lowByte) > 0; +#else + return lowByte.Compare(highByte) > 0 && highByte.Compare(lowByte) < 0; +#endif + } + + template + void CheckConstexprBitMap() { + constexpr auto bits = [] { + TBitMap<131, TChunk> result; + result.Set(0).Set(63).Set(130); + result[64] = result[63]; + result <<= 1; + result.Reset(1); + return result; + }(); + static_assert(bits == TBitMap<131, TChunk>().Set(64).Set(65)); + static_assert(THash>{}(bits) == bits.Hash()); + + static_assert(TestConstexprConversion()); + static_assert(TestConstexprConversion()); + static_assert(TestConstexprConversion()); + static_assert(TestConstexprConversion()); + static_assert(TestConstexprExport()); + static_assert(TestConstexprExport()); + static_assert(TestConstexprExport()); + static_assert(TestConstexprExport()); + + UNIT_ASSERT((TestConstexprConversion())); + UNIT_ASSERT((TestConstexprConversion())); + UNIT_ASSERT((TestConstexprConversion())); + UNIT_ASSERT((TestConstexprConversion())); + UNIT_ASSERT((TestConstexprExport())); + UNIT_ASSERT((TestConstexprExport())); + UNIT_ASSERT((TestConstexprExport())); + UNIT_ASSERT((TestConstexprExport())); + } +} // namespace + Y_UNIT_TEST_SUITE(TBitMapTest) { + Y_UNIT_TEST(TestConstexpr) { + static_assert(TestConstexprCompare()); + UNIT_ASSERT(TestConstexprCompare()); + CheckConstexprBitMap(); + CheckConstexprBitMap(); + CheckConstexprBitMap(); + CheckConstexprBitMap(); + } + Y_UNIT_TEST(TestBitMap) { TBitMap<101> bitmap; diff --git a/util/str_stl.h b/util/str_stl.h index 4bbcd928b4a..c709935ef6a 100644 --- a/util/str_stl.h +++ b/util/str_stl.h @@ -39,7 +39,7 @@ namespace NHashPrivate { struct THashHelper { using is_default_implementation = std::true_type; - inline size_t operator()(const T& t) const noexcept { + constexpr size_t operator()(const T& t) const noexcept { return (size_t)t; // If you have a compilation error here, look at explanation below: // Probably error is caused by undefined template specialization of THash // You can find examples of specialization in this file From aec1437947ca6ee993d08ce9e45fc1b1f93d3866 Mon Sep 17 00:00:00 2001 From: Evgeniy Ivanov Date: Thu, 5 Nov 2026 13:52:11 +0000 Subject: [PATCH 11/19] C++ SDK: optional pile connectivity report in the bridge GetClusterState (#58006) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index d6e73becde9..219f59a7b09 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -c989daddb9374148beb8f9b21797747a2ef66d85 +fb1534204f2eb471dbe3220cfca995152dc1d1c5 diff --git a/CHANGELOG.md b/CHANGELOG.md index a5ae6bfd6a7..f63a0e34621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Added `TGetClusterStateSettings::IncludePileConnectivity()` and `TGetClusterStateResult::GetPileConnectivity()` for optional bridge static-node connectivity reports. + * Added `TUpdateClusterStateSettings::ExpectedGeneration()` to apply a bridge cluster state update only if the cluster state still has the given generation. * Added draft `TBridgeClient::SetDatabaseBridgeMode()` to atomically change a database's bridge mode. `TSetDatabaseBridgeModeSettings::RetainPileName()` selects the pile to retain when disabling bridge mode. From 6143805f2a2b2bcb96d2df25fff9bb72679cb736 Mon Sep 17 00:00:00 2001 From: Nikolay Shestakov Date: Thu, 5 Nov 2026 13:52:20 +0000 Subject: [PATCH 12/19] Fix Topic producer retries after the producer is destroyed (#58042) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + src/client/topic/impl/producer.cpp | 24 +-- src/client/topic/impl/producer.h | 9 +- .../topic/ut/producer_retry_policy_ut.cpp | 149 ++++++++++++++++++ 5 files changed, 172 insertions(+), 14 deletions(-) create mode 100644 src/client/topic/ut/producer_retry_policy_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 219f59a7b09..d97526b7b12 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -fb1534204f2eb471dbe3220cfca995152dc1d1c5 +c192e31a6d4e98cdf056f1b0fe1f22a42ee9ecf0 diff --git a/CHANGELOG.md b/CHANGELOG.md index f63a0e34621..aa941f0b777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Fixed PersQueue topic settings losing replication-consumer and consumer-offset-mirroring attributes when copied from DescribeTopic into CreateTopic or AlterTopic settings. +* Fixed Topic producer retries touching the producer after it was destroyed. Write sessions keep a snapshot of the retry policy and no longer read producer settings from the gRPC thread. + * Supported Topic read sessions counting replication control messages as data offsets, which could block commits after an offset gap. ## v3.26.0 diff --git a/src/client/topic/impl/producer.cpp b/src/client/topic/impl/producer.cpp index b616c87d284..3cb609be088 100644 --- a/src/client/topic/impl/producer.cpp +++ b/src/client/topic/impl/producer.cpp @@ -1527,35 +1527,37 @@ void TProducer::TMessagesWorker::RebuildPendingMessagesIndex(std::uint32_t parti //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TProducer::TProducerRetryPolicy -TProducer::TProducerRetryPolicy::TProducerRetryPolicy(TProducer* producer) - : Producer(producer) +TProducer::TProducerRetryPolicy::TProducerRetryPolicy(::IRetryPolicy::TPtr userPolicy) + : UserPolicy(userPolicy ? std::move(userPolicy) : NYdb::NTopic::IRetryPolicy::GetDefaultPolicy()) {} typename TProducer::TProducerRetryPolicy::IRetryState::TPtr TProducer::TProducerRetryPolicy::CreateRetryState() const { struct TRetryState : public IRetryState { - TRetryState(TProducer* producer) - : Producer(producer) + explicit TRetryState(::IRetryPolicy::TPtr userPolicy) + : UserPolicy(std::move(userPolicy)) {} - ~TRetryState() = default; + TMaybe GetNextRetryDelay(EStatus status) override { + // OVERLOADED closes this session so the producer can switch partitions + // (HandleAutoPartitioning). A user retry would keep the session on the + // inactive partition, so the user policy must not see this status. if (status == EStatus::OVERLOADED) { return Nothing(); } if (!UserRetryState) { - auto policy = Producer->Settings.RetryPolicy_ ? Producer->Settings.RetryPolicy_ : NYdb::NTopic::IRetryPolicy::GetDefaultPolicy(); - UserRetryState = policy->CreateRetryState(); + UserRetryState = UserPolicy->CreateRetryState(); } return UserRetryState->GetNextRetryDelay(status); } private: - TProducer* Producer; + ::IRetryPolicy::TPtr UserPolicy; IRetryState::TPtr UserRetryState; }; - - return std::make_unique(Producer); + + return std::make_unique(UserPolicy); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1767,7 +1769,7 @@ TProducer::TProducer( SessionsWorker = std::make_shared(this); MessagesWorker = std::make_shared(this); EventsWorker = std::make_shared(this); - RetryPolicy = std::make_shared(this); + RetryPolicy = std::make_shared(Settings.RetryPolicy_); if (Settings.AsyncExecutionMode_) { MainWorkerThread = SystemThreadFactory()->Run([this] { diff --git a/src/client/topic/impl/producer.h b/src/client/topic/impl/producer.h index 4ee678baaf0..644cf72a27c 100644 --- a/src/client/topic/impl/producer.h +++ b/src/client/topic/impl/producer.h @@ -18,6 +18,7 @@ namespace NYdb::inline V3::NTopic { struct TProducerMessageInfoTestHelper; +struct TProducerRetryPolicyTestHelper; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TProducer @@ -26,6 +27,7 @@ class TProducer : public IProducer, public TContinuationTokenIssuer, public std::enable_shared_from_this { friend struct TProducerMessageInfoTestHelper; + friend struct TProducerRetryPolicyTestHelper; private: static constexpr size_t MAX_EPOCH = 1'000'000'000; @@ -136,16 +138,19 @@ class TProducer : public IProducer, //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Custom retry policy + // Write-session settings keep this object alive. GetNextRetryDelay may run + // on the gRPC thread after TProducer is destroyed, so the user policy is + // stored here instead of a producer pointer. struct TProducerRetryPolicy : public ::IRetryPolicy { using TSelf = TProducerRetryPolicy; using TPtr = std::shared_ptr; - TProducerRetryPolicy(TProducer* producer); + explicit TProducerRetryPolicy(::IRetryPolicy::TPtr userPolicy); ~TProducerRetryPolicy() = default; typename IRetryState::TPtr CreateRetryState() const override; private: - TProducer* Producer; + ::IRetryPolicy::TPtr UserPolicy; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/client/topic/ut/producer_retry_policy_ut.cpp b/src/client/topic/ut/producer_retry_policy_ut.cpp new file mode 100644 index 00000000000..b5227918aec --- /dev/null +++ b/src/client/topic/ut/producer_retry_policy_ut.cpp @@ -0,0 +1,149 @@ +#include +#include + +#include + +#include + +using namespace NYdb; +using namespace NYdb::NTopic; + +namespace NYdb::inline V3::NTopic { + +struct TProducerRetryPolicyTestHelper { + static TProducer::TProducerRetryPolicy::TPtr Create(IRetryPolicy::TPtr userPolicy) { + return std::make_shared(std::move(userPolicy)); + } + + static TProducer::TProducerRetryPolicy::TPtr FromProducer(IProducer& producer) { + return static_cast(producer).RetryPolicy; + } +}; + +} // namespace NYdb::inline V3::NTopic + +namespace { + + class TScriptedRetryPolicy: public NTopic::IRetryPolicy { + public: + mutable int StateCount = 0; + mutable int CallCount = 0; + mutable EStatus LastStatus = EStatus::STATUS_UNDEFINED; + + struct TState: public NTopic::IRetryPolicy::IRetryState { + explicit TState(TScriptedRetryPolicy* policy) + : Policy(policy) + { + } + + TMaybe GetNextRetryDelay(EStatus status) override { + ++Policy->CallCount; + Policy->LastStatus = status; + if (AttemptsLeft == 0) { + return Nothing(); + } + --AttemptsLeft; + return TDuration::MilliSeconds(15); + } + + TScriptedRetryPolicy* Policy; + int AttemptsLeft = 1; + }; + + NTopic::IRetryPolicy::IRetryState::TPtr CreateRetryState() const override { + ++StateCount; + return std::make_unique(const_cast(this)); + } + }; + +} // namespace + +Y_UNIT_TEST_SUITE(ProducerRetryPolicy) { + + Y_UNIT_TEST(OverloadedDoesNotConsultUserPolicy) { + auto user = std::make_shared(); + auto policy = TProducerRetryPolicyTestHelper::Create(user); + auto state = policy->CreateRetryState(); + + const auto delay = state->GetNextRetryDelay(EStatus::OVERLOADED); + + UNIT_ASSERT(!delay); + UNIT_ASSERT_VALUES_EQUAL(user->StateCount, 0); + UNIT_ASSERT_VALUES_EQUAL(user->CallCount, 0); + } + + Y_UNIT_TEST(DelegatesOtherStatusesAndReusesState) { + auto user = std::make_shared(); + auto policy = TProducerRetryPolicyTestHelper::Create(user); + auto state = policy->CreateRetryState(); + + const auto first = state->GetNextRetryDelay(EStatus::UNAVAILABLE); + UNIT_ASSERT(first); + UNIT_ASSERT_VALUES_EQUAL(first->MilliSeconds(), 15); + UNIT_ASSERT_VALUES_EQUAL(user->StateCount, 1); + UNIT_ASSERT_VALUES_EQUAL(user->CallCount, 1); + UNIT_ASSERT(user->LastStatus == EStatus::UNAVAILABLE); + + // OVERLOADED is handled by the wrapper and must not consume a user attempt. + UNIT_ASSERT(!state->GetNextRetryDelay(EStatus::OVERLOADED)); + UNIT_ASSERT_VALUES_EQUAL(user->CallCount, 1); + + const auto second = state->GetNextRetryDelay(EStatus::UNAVAILABLE); + UNIT_ASSERT(!second); + UNIT_ASSERT_VALUES_EQUAL(user->StateCount, 1); + UNIT_ASSERT_VALUES_EQUAL(user->CallCount, 2); + } + + Y_UNIT_TEST(RetryStatesAreIndependent) { + auto user = std::make_shared(); + auto policy = TProducerRetryPolicyTestHelper::Create(user); + auto first = policy->CreateRetryState(); + auto second = policy->CreateRetryState(); + + UNIT_ASSERT(first->GetNextRetryDelay(EStatus::UNAVAILABLE)); + UNIT_ASSERT(!first->GetNextRetryDelay(EStatus::UNAVAILABLE)); + UNIT_ASSERT(second->GetNextRetryDelay(EStatus::TRANSPORT_UNAVAILABLE)); + UNIT_ASSERT_VALUES_EQUAL(user->StateCount, 2); + UNIT_ASSERT(user->LastStatus == EStatus::TRANSPORT_UNAVAILABLE); + } + + Y_UNIT_TEST(SurvivesProducerDestruction) { + NTests::TTopicSdkTestSetup setup{TEST_CASE_NAME, NTests::TTopicSdkTestSetup::MakeServerSettings(), false}; + setup.CreateTopic(); + + auto client = setup.MakeClient(); + TProducerSettings settings; + settings + .Path(setup.GetTopicPath()) + .Codec(ECodec::RAW); + settings.ProducerIdPrefix("retry-after-destroy"); + settings.PartitionChooserStrategy(TProducerSettings::EPartitionChooserStrategy::KafkaHash); + auto user = std::make_shared(); + settings.RetryPolicy(user); + + auto producer = client.CreateProducer(settings); + auto policy = TProducerRetryPolicyTestHelper::FromProducer(*producer); + producer.reset(); + + const auto delay = policy->CreateRetryState()->GetNextRetryDelay(EStatus::UNAVAILABLE); + UNIT_ASSERT(delay); + UNIT_ASSERT_VALUES_EQUAL(delay->MilliSeconds(), 15); + UNIT_ASSERT_VALUES_EQUAL(user->CallCount, 1); + } + + Y_UNIT_TEST(MissingUserPolicyUsesDefaultAndDoesNotNeedProducer) { + // The wrapper must stay usable with no TProducer: write sessions call it + // after the producer has already been destroyed. + auto policy = TProducerRetryPolicyTestHelper::Create(nullptr); + auto state = policy->CreateRetryState(); + + const auto retry = state->GetNextRetryDelay(EStatus::UNAVAILABLE); + UNIT_ASSERT(retry); + UNIT_ASSERT(*retry > TDuration::Zero()); + + auto fatal = policy->CreateRetryState(); + UNIT_ASSERT(!fatal->GetNextRetryDelay(EStatus::SCHEME_ERROR)); + UNIT_ASSERT(!fatal->GetNextRetryDelay(EStatus::OVERLOADED)); + } + +} // Y_UNIT_TEST_SUITE(ProducerRetryPolicy) From 572200d9d9bc0240a1a155df5b58974d78881240 Mon Sep 17 00:00:00 2001 From: Ivanov Victor Date: Thu, 5 Nov 2026 13:52:28 +0000 Subject: [PATCH 13/19] Recognize LBK federation endpoints in C++ SDK (#58227) --- .github/last_commit.txt | 2 +- .../impl/write_session_impl.cpp | 4 ++- .../persqueue_public/include/write_session.h | 2 +- .../persqueue_public/ut_cds/endpoint_ut.cpp | 15 +++++++++++ src/client/topic/impl/producer.cpp | 4 ++- .../ut_producer_endpoint/endpoint_ut.cpp | 25 +++++++++++++++++++ 6 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 src/client/persqueue_public/ut_cds/endpoint_ut.cpp create mode 100644 src/client/topic/ut_producer_endpoint/endpoint_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index d97526b7b12..e90d5b93fd1 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -c192e31a6d4e98cdf056f1b0fe1f22a42ee9ecf0 +88b0be89db4f5803aa92bd241964fcd2fa77c213 diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index 1e2141a2450..49e8fb74ba8 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -118,7 +118,9 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat bool IsFederation(const std::string& endpoint) { std::string_view host = GetHost(endpoint); - return host == "logbroker.yandex.net" || host == "logbroker-prestable.yandex.net"; + return host == "logbroker.yandex.net" || + host == "logbroker-prestable.yandex.net" || + host.ends_with(".lbk.logbroker.yandex.net"); } void TWriteSessionImpl::DoCdsRequest(TDuration delay) { diff --git a/src/client/persqueue_public/include/write_session.h b/src/client/persqueue_public/include/write_session.h index 29a3ec69ee5..1fdaeec414b 100644 --- a/src/client/persqueue_public/include/write_session.h +++ b/src/client/persqueue_public/include/write_session.h @@ -11,7 +11,7 @@ namespace NYdb::inline V3::NPersQueue { enum class EClusterDiscoveryMode { - Auto = 0, // enables cluster discovery only for hostname "logbroker.yandex.net" and "logbroker-prestable.yandex.net" + Auto = 0, // Enables discovery for main, prestable and LBK federation endpoints. On, Off }; diff --git a/src/client/persqueue_public/ut_cds/endpoint_ut.cpp b/src/client/persqueue_public/ut_cds/endpoint_ut.cpp new file mode 100644 index 00000000000..1543c528251 --- /dev/null +++ b/src/client/persqueue_public/ut_cds/endpoint_ut.cpp @@ -0,0 +1,15 @@ +#include + +#include + +namespace NYdb::inline V3::NPersQueue { + bool IsFederation(const std::string& endpoint); +} // namespace NYdb::inline V3::NPersQueue + +Y_UNIT_TEST_SUITE(PersQueueFederationEndpointTest) { + Y_UNIT_TEST(RecognizesLbkSubdomain) { + UNIT_ASSERT(NYdb::Dev::NPersQueue::IsFederation("service.lbk.logbroker.yandex.net:2135")); + UNIT_ASSERT(!NYdb::Dev::NPersQueue::IsFederation("notlbk.logbroker.yandex.net:2135")); + UNIT_ASSERT(!NYdb::Dev::NPersQueue::IsFederation("service.lbk.logbroker.yandex.net.example.org:2135")); + } +} // Y_UNIT_TEST_SUITE(PersQueueFederationEndpointTest) diff --git a/src/client/topic/impl/producer.cpp b/src/client/topic/impl/producer.cpp index 3cb609be088..e77a8ee5be8 100644 --- a/src/client/topic/impl/producer.cpp +++ b/src/client/topic/impl/producer.cpp @@ -2032,7 +2032,9 @@ void TProducer::RunUserEventLoop() { bool TProducer::IsFederation(const std::string& endpoint) { std::string_view host = GetHost(endpoint); - return host == "logbroker.yandex.net" || host == "logbroker-prestable.yandex.net"; + return host == "logbroker.yandex.net" || + host == "logbroker-prestable.yandex.net" || + host.ends_with(".lbk.logbroker.yandex.net"); } void TProducer::GetSessionClosedEventAndDie(WrappedWriteSessionPtr wrappedSession, std::optional sessionClosedEvent) { diff --git a/src/client/topic/ut_producer_endpoint/endpoint_ut.cpp b/src/client/topic/ut_producer_endpoint/endpoint_ut.cpp new file mode 100644 index 00000000000..c52e53440c3 --- /dev/null +++ b/src/client/topic/ut_producer_endpoint/endpoint_ut.cpp @@ -0,0 +1,25 @@ +#include +#include +#include +#include + +#include + +namespace NYdb::inline V3::NTopic { + Y_UNIT_TEST_SUITE(TopicProducerFederationEndpointTest) { + Y_UNIT_TEST(RejectsLbkSubdomain) { + TDriverConfig config; + config.SetEndpoint("service.lbk.logbroker.yandex.net:2135"); + config.SetDatabase("/Root"); + config.SetDiscoveryMode(EDiscoveryMode::Off); + TDriver driver(config); + TTopicClient client(driver); + TProducerSettings settings; + settings.Path("test-topic"); + settings.ProducerIdPrefix("test-producer"); + UNIT_ASSERT_EXCEPTION_CONTAINS( + client.CreateProducer(settings), TContractViolation, + "Producer is not supported for federation"); + } + } // Y_UNIT_TEST_SUITE(TopicProducerFederationEndpointTest) +} // namespace NYdb::inline V3::NTopic From e952baf08d7fdfdc1f2abe8d231bf322e4a8bdb5 Mon Sep 17 00:00:00 2001 From: kulaad Date: Thu, 5 Nov 2026 13:52:37 +0000 Subject: [PATCH 14/19] Describe GLOBAL replication transactions (#57214) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 ++ src/api/protos/draft/ydb_replication.proto | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index e90d5b93fd1..73c77845b63 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -88b0be89db4f5803aa92bd241964fcd2fa77c213 +5c5df6a4773b1dd84742c1f0a4867be9fa284c7e diff --git a/CHANGELOG.md b/CHANGELOG.md index aa941f0b777..2a585d9da8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Added optional `TReplicationDescription::GetTransactions()` to report assigned global-consistency replication transactions and versions pending transaction-ID assignment. + * Added `TGetClusterStateSettings::IncludePileConnectivity()` and `TGetClusterStateResult::GetPileConnectivity()` for optional bridge static-node connectivity reports. * Added `TUpdateClusterStateSettings::ExpectedGeneration()` to apply a bridge cluster state update only if the cluster state still has the given generation. diff --git a/src/api/protos/draft/ydb_replication.proto b/src/api/protos/draft/ydb_replication.proto index d2f55196d26..27d4757e035 100644 --- a/src/api/protos/draft/ydb_replication.proto +++ b/src/api/protos/draft/ydb_replication.proto @@ -3,6 +3,7 @@ option cc_enable_arenas = true; import "src/api/protos/annotations/validation.proto"; import "src/api/protos/ydb_issue_message.proto"; +import "src/api/protos/ydb_common.proto"; import "src/api/protos/ydb_operation.proto"; import "src/api/protos/ydb_scheme.proto"; @@ -62,6 +63,21 @@ message AvailabilityPeriod { } message DescribeReplicationResult { + message Transactions { + message Assigned { + // Exclusive upper source version of the replication commit interval. + VirtualTimestamp upper_bound_barrier = 1; + // Assigned write transaction ID on the destination. + uint64 write_tx_id = 2; + } + + // Assigned transactions whose commit has not yet completed. + repeated Assigned assigned = 1; + // Distinct versions queued for transaction-ID assignment, not transaction IDs. + // This in-memory queue is rebuilt from worker requests after restart. + repeated VirtualTimestamp pending = 2; + } + message Stats { optional google.protobuf.Duration lag = 1; optional float initial_scan_progress = 2; @@ -105,6 +121,10 @@ message DescribeReplicationResult { DoneState done = 6; PausedState paused = 9; } + // Current global-consistency transaction state, ordered by interval version. + // Reported independently of include_stats; absent when unavailable or in row mode. + // Empty lists do not prove that replication caught up with the source. + Transactions transactions = 11; } message DescribeTransferRequest { From baebf9c307ba52b854dfaafacb8ff24ebc4335f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:52:38 +0000 Subject: [PATCH 15/19] Update import generation: 53 --- .github/import_generation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/import_generation.txt b/.github/import_generation.txt index 59343b09ec7..fb1e7bc8699 100644 --- a/.github/import_generation.txt +++ b/.github/import_generation.txt @@ -1 +1 @@ -53 +54 From c76642801b6dca59e7d72da2592ce71fab9ff5af Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:54:14 +0300 Subject: [PATCH 16/19] Register imported standalone SDK tests --- tests/unit/client/CMakeLists.txt | 13 +++++++++++++ tests/unit/client/topic/CMakeLists.txt | 24 +++++++++++++++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/unit/client/CMakeLists.txt b/tests/unit/client/CMakeLists.txt index 75a5dbb4cf4..845811ad0ca 100644 --- a/tests/unit/client/CMakeLists.txt +++ b/tests/unit/client/CMakeLists.txt @@ -2,6 +2,19 @@ add_subdirectory(federated_topic) add_subdirectory(oauth2_token_exchange/helpers) add_subdirectory(topic) +add_ydb_test(NAME client-persqueue-replication-attributes_ut + INCLUDE_DIRS ${YDB_SDK_SOURCE_DIR} + SOURCES ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/ut/control_plane/replication_attributes_ut.cpp + LINK_LIBRARIES cpp-client-ydb_persqueue_public + LABELS unit +) + +add_ydb_test(NAME client-persqueue-federation-endpoint_ut + SOURCES ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/ut_cds/endpoint_ut.cpp + LINK_LIBRARIES cpp-client-ydb_persqueue_public + LABELS unit +) + # src/client/persqueue_public/ut/compress_executor_ut.cpp requires the # upstream TPersQueueYdbSdkTestSetup and ydb/core server fixtures, which are # absent from the standalone SDK repository. diff --git a/tests/unit/client/topic/CMakeLists.txt b/tests/unit/client/topic/CMakeLists.txt index 65134cb4fb9..04eb675bbda 100644 --- a/tests/unit/client/topic/CMakeLists.txt +++ b/tests/unit/client/topic/CMakeLists.txt @@ -43,6 +43,24 @@ add_ydb_test(NAME client-topic-read-session-accounting_ut LABELS unit ) -# read_session_kafka_timestamps_ut.cpp, reset_offset_ut.cpp, and -# write_session_connect_ut.cpp use the upstream TTopicSdkTestSetup and -# ydb/core server fixtures, which are absent from the standalone SDK repository. +add_ydb_test(NAME client-topic-read-session-replication_ut + INCLUDE_DIRS ${YDB_SDK_SOURCE_DIR} + SOURCES ${YDB_SDK_SOURCE_DIR}/src/client/topic/ut/replication/read_session_replication_ut.cpp + LINK_LIBRARIES + YDB-CPP-SDK::Topic + api-grpc + cpp-testing-common + containers-disjoint_interval_tree + LABELS unit +) + +add_ydb_test(NAME client-topic-producer-endpoint_ut + SOURCES ${YDB_SDK_SOURCE_DIR}/src/client/topic/ut_producer_endpoint/endpoint_ut.cpp + LINK_LIBRARIES YDB-CPP-SDK::Topic + LABELS unit +) + +# producer_retry_policy_ut.cpp, read_session_kafka_timestamps_ut.cpp, +# reset_offset_ut.cpp, and write_session_connect_ut.cpp use the upstream +# TTopicSdkTestSetup and ydb/core server fixtures, which are absent from the +# standalone SDK repository. From ca801ef174e436c1030a039ccf9fc196c2f6cdf0 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:14:37 +0300 Subject: [PATCH 17/19] Translate qualified SDK namespaces during import --- .github/scripts/copy_sources.sh | 1 + scripts/test_copy_sources_plugins.sh | 3 +++ src/client/persqueue_public/ut_cds/endpoint_ut.cpp | 6 +++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/scripts/copy_sources.sh b/.github/scripts/copy_sources.sh index 89d0537cbd7..3bd7df545ab 100755 --- a/.github/scripts/copy_sources.sh +++ b/.github/scripts/copy_sources.sh @@ -268,6 +268,7 @@ do s/'$SED_INCLUDE'ydb\/public\/sdk\/cpp\//\1\2/g; s/'$SED_INCLUDE'library\/cpp\/retry\/retry_policy\.h/\1\2ydb-cpp-sdk\/library\/retry\/retry_policy\.h/g; s/'$SED_INCLUDE'library\/cpp\/string_utils\/base64\/base64\.h/\1\2src\/library\/string_utils\/base64\/base64\.h/g; + s/NYdb::Dev::/NYdb::V3::/g; s/(inline\s*)Dev/\1V3/g; s/(inline\s*namespace\s*)Dev/\1V3/g;' $f done diff --git a/scripts/test_copy_sources_plugins.sh b/scripts/test_copy_sources_plugins.sh index 7a632d7f8f9..6087ed534e3 100755 --- a/scripts/test_copy_sources_plugins.sh +++ b/scripts/test_copy_sources_plugins.sh @@ -39,6 +39,7 @@ mkdir -p "$MONOREPO/tools/enum_parser" echo "MONOREPO_TRACE" > "$MONOREPO/ydb/public/sdk/cpp/plugins/trace/otel/trace.cpp" echo "MONOREPO_METRICS" > "$MONOREPO/ydb/public/sdk/cpp/plugins/metrics/otel/metrics.cpp" +printf 'namespace NYdb::inline Dev { struct TProbe {}; }\nusing TImportedProbe = NYdb::Dev::TProbe;\n' > "$MONOREPO/ydb/public/sdk/cpp/src/client/qualified_namespace.cpp" echo "MONOREPO_AGENTS" > "$MONOREPO/ydb/public/sdk/cpp/AGENTS.md" echo "# MONOREPO_PLUGINS_CMAKE" > "$MONOREPO/ydb/public/sdk/cpp/plugins/CMakeLists.txt" echo "// monorepo type_switcher" > "$MONOREPO/ydb/public/sdk/cpp/include/ydb-cpp-sdk/type_switcher.h" @@ -144,6 +145,8 @@ assert_contains() { assert_contains "$OSS/plugins/trace/otel/trace.cpp" "MONOREPO_TRACE" "OSS_TRACE" assert_contains "$OSS/plugins/metrics/otel/metrics.cpp" "MONOREPO_METRICS" "OSS_METRICS" +assert_contains "$OSS/src/client/qualified_namespace.cpp" "inline V3" "inline Dev" +assert_contains "$OSS/src/client/qualified_namespace.cpp" "NYdb::V3::TProbe" "NYdb::Dev::" assert_contains "$OSS/AGENTS.md" "OSS_AGENTS" "MONOREPO_AGENTS" assert_contains "$OSS/tests/slo_workloads/key_value/main.cpp" "OSS_SLO_WORKLOAD" "" assert_contains "$OSS/tests/deb_package/Dockerfile" "OSS_DEB_PACKAGE" "" diff --git a/src/client/persqueue_public/ut_cds/endpoint_ut.cpp b/src/client/persqueue_public/ut_cds/endpoint_ut.cpp index 1543c528251..5c2ea5d51f5 100644 --- a/src/client/persqueue_public/ut_cds/endpoint_ut.cpp +++ b/src/client/persqueue_public/ut_cds/endpoint_ut.cpp @@ -8,8 +8,8 @@ namespace NYdb::inline V3::NPersQueue { Y_UNIT_TEST_SUITE(PersQueueFederationEndpointTest) { Y_UNIT_TEST(RecognizesLbkSubdomain) { - UNIT_ASSERT(NYdb::Dev::NPersQueue::IsFederation("service.lbk.logbroker.yandex.net:2135")); - UNIT_ASSERT(!NYdb::Dev::NPersQueue::IsFederation("notlbk.logbroker.yandex.net:2135")); - UNIT_ASSERT(!NYdb::Dev::NPersQueue::IsFederation("service.lbk.logbroker.yandex.net.example.org:2135")); + UNIT_ASSERT(NYdb::V3::NPersQueue::IsFederation("service.lbk.logbroker.yandex.net:2135")); + UNIT_ASSERT(!NYdb::V3::NPersQueue::IsFederation("notlbk.logbroker.yandex.net:2135")); + UNIT_ASSERT(!NYdb::V3::NPersQueue::IsFederation("service.lbk.logbroker.yandex.net.example.org:2135")); } } // Y_UNIT_TEST_SUITE(PersQueueFederationEndpointTest) From 83dc7745ba9f1a484563fdd500f8cdd52e40145e Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:36:56 +0300 Subject: [PATCH 18/19] Preserve import scripts and link PersQueue Topic dependencies --- .github/scripts/copy_sources.sh | 2 +- scripts/test_copy_sources_plugins.sh | 5 +++++ src/client/persqueue_public/impl/CMakeLists.txt | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/scripts/copy_sources.sh b/.github/scripts/copy_sources.sh index 3bd7df545ab..0323067c0fb 100755 --- a/.github/scripts/copy_sources.sh +++ b/.github/scripts/copy_sources.sh @@ -259,7 +259,7 @@ rm -rf $tmp_dir/adapters $tmp_dir/client SED_INCLUDE='(^\s*#include\s*)(<|\")' -find $tmp_dir -type f -regex ".*\(h\|cpp\|cpp.in\|c\|ipp\|jnj\|rl6\|h.txt\|proto\)$" | while read f; +find $tmp_dir -type f -regex ".*\.\(h\|cpp\|cpp\.in\|c\|ipp\|jnj\|rl6\|h\.txt\|proto\)$" | while read f; do sed -i -E \ 's/'$SED_INCLUDE'ydb\/public\/api\//\1\2src\/api\//g; diff --git a/scripts/test_copy_sources_plugins.sh b/scripts/test_copy_sources_plugins.sh index 6087ed534e3..e7f01353aa4 100755 --- a/scripts/test_copy_sources_plugins.sh +++ b/scripts/test_copy_sources_plugins.sh @@ -69,6 +69,7 @@ mkdir -p "$OSS/library/cpp/example" mkdir -p "$OSS/contrib/libs/libc_compat" mkdir -p "$OSS/tools/enum_parser" mkdir -p "$OSS/.git" +mkdir -p "$OSS/.github/scripts" mkdir -p "$OSS/plugins/trace/otel" mkdir -p "$OSS/plugins/metrics/otel" mkdir -p "$OSS/include/ydb-cpp-sdk" @@ -88,6 +89,8 @@ touch "$OSS/.gitignore" "$OSS/CMakePresets.json" "$OSS/CMakeLists.txt" touch "$OSS/codecov.yml" touch "$OSS/LICENSE" "$OSS/README.md" echo "OSS_AGENTS" > "$OSS/AGENTS.md" +echo '# NYdb::Dev::TProbe' > "$OSS/.github/scripts/namespace.sh" +echo '# NYdb::Dev::TProbe' > "$OSS/scripts/namespace.sh" touch "$OSS/tests/slo_workloads/.dockerignore" "$OSS/tests/slo_workloads/Dockerfile" echo "OSS_SLO_WORKLOAD" > "$OSS/tests/slo_workloads/key_value/main.cpp" echo "OSS_DEB_PACKAGE" > "$OSS/tests/deb_package/Dockerfile" @@ -147,6 +150,8 @@ assert_contains "$OSS/plugins/trace/otel/trace.cpp" "MONOREPO_TRACE" "OSS_TRACE" assert_contains "$OSS/plugins/metrics/otel/metrics.cpp" "MONOREPO_METRICS" "OSS_METRICS" assert_contains "$OSS/src/client/qualified_namespace.cpp" "inline V3" "inline Dev" assert_contains "$OSS/src/client/qualified_namespace.cpp" "NYdb::V3::TProbe" "NYdb::Dev::" +assert_contains "$OSS/.github/scripts/namespace.sh" "NYdb::Dev::TProbe" "NYdb::V3::" +assert_contains "$OSS/scripts/namespace.sh" "NYdb::Dev::TProbe" "NYdb::V3::" assert_contains "$OSS/AGENTS.md" "OSS_AGENTS" "MONOREPO_AGENTS" assert_contains "$OSS/tests/slo_workloads/key_value/main.cpp" "OSS_SLO_WORKLOAD" "" assert_contains "$OSS/tests/deb_package/Dockerfile" "OSS_DEB_PACKAGE" "" diff --git a/src/client/persqueue_public/impl/CMakeLists.txt b/src/client/persqueue_public/impl/CMakeLists.txt index 2f8bab2ef77..f82f1616129 100644 --- a/src/client/persqueue_public/impl/CMakeLists.txt +++ b/src/client/persqueue_public/impl/CMakeLists.txt @@ -12,6 +12,8 @@ target_link_libraries(client-ydb_persqueue_public-impl PUBLIC impl-internal-make_request client-ydb_common_client-impl client-ydb_driver + client-ydb_topic-common + client-ydb_topic-impl client-ydb_persqueue_public-include string_utils-misc ) From 18bec07a87f6b1bd0a7ebb708e78611601ba0189 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:18:23 +0300 Subject: [PATCH 19/19] Wait for table session cleanup in integration tests --- tests/integration/sessions/main.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/sessions/main.cpp b/tests/integration/sessions/main.cpp index c451f3997dd..14df461519a 100644 --- a/tests/integration/sessions/main.cpp +++ b/tests/integration/sessions/main.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -52,6 +53,15 @@ void WarmPoolCreateSession(NYdb::NQuery::TQueryClient& client, std::string& sess ASSERT_EQ(resultSet.ColumnsCount(), 2u); } + void WaitForNoActiveSessions(const NYdb::NTable::TTableClient& client) { + // A ready query future does not imply that its callback has released + // the session it captured. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (client.GetActiveSessionCount() != 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + } + void WaitForSessionsInPool(NYdb::NQuery::TQueryClient& client, std::int64_t expected) { int attempt = 10; while (attempt--) { @@ -145,6 +155,7 @@ TEST(YdbSdkSessions, TestMultipleSessions) { sids.clear(); results.clear(); + WaitForNoActiveSessions(client); ASSERT_EQ(client.GetActiveSessionCount(), 0); driver.Stop(true); @@ -191,6 +202,7 @@ TEST(YdbSdkSessions, TestActiveSessionCountAfterBadSession) { sids.clear(); results.clear(); + WaitForNoActiveSessions(client); ASSERT_EQ(client.GetActiveSessionCount(), 0); driver.Stop(true);