From 91e3668ff8483f8e82c7584e928ef74b0d018e33 Mon Sep 17 00:00:00 2001 From: YDBot Date: Wed, 11 Nov 2026 17:06:55 +0000 Subject: [PATCH 01/17] Merge pull request #58102 from ydb-platform/robots/branch-sync/merge-rightlib-261104-0045 --- .github/last_commit.txt | 2 +- tests/integration/sessions/main.cpp | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 73c77845b63..1b12ffc7105 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -5c5df6a4773b1dd84742c1f0a4867be9fa284c7e +7c7cbc3873cc519ce596389242e391f8b327a4e4 diff --git a/tests/integration/sessions/main.cpp b/tests/integration/sessions/main.cpp index 14df461519a..c451f3997dd 100644 --- a/tests/integration/sessions/main.cpp +++ b/tests/integration/sessions/main.cpp @@ -11,7 +11,6 @@ #include #include -#include #include #include @@ -53,15 +52,6 @@ void WarmPoolCreateSession(NYdb::NQuery::TQueryClient& client, std::string& sess ASSERT_EQ(resultSet.ColumnsCount(), 2u); } - void WaitForNoActiveSessions(const NYdb::NTable::TTableClient& client) { - // A ready query future does not imply that its callback has released - // the session it captured. - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - while (client.GetActiveSessionCount() != 0 && std::chrono::steady_clock::now() < deadline) { - std::this_thread::yield(); - } - } - void WaitForSessionsInPool(NYdb::NQuery::TQueryClient& client, std::int64_t expected) { int attempt = 10; while (attempt--) { @@ -155,7 +145,6 @@ TEST(YdbSdkSessions, TestMultipleSessions) { sids.clear(); results.clear(); - WaitForNoActiveSessions(client); ASSERT_EQ(client.GetActiveSessionCount(), 0); driver.Stop(true); @@ -202,7 +191,6 @@ TEST(YdbSdkSessions, TestActiveSessionCountAfterBadSession) { sids.clear(); results.clear(); - WaitForNoActiveSessions(client); ASSERT_EQ(client.GetActiveSessionCount(), 0); driver.Stop(true); From 50426cd30c75140ef69f31835b8b8dd577055dd3 Mon Sep 17 00:00:00 2001 From: YDBot Date: Wed, 11 Nov 2026 17:07:02 +0000 Subject: [PATCH 02/17] Merge pull request #58403 from ydb-platform/robots/branch-sync/merge-rightlib-261106-0931 --- .github/last_commit.txt | 2 +- library/cpp/yt/rseq/benchmarks/cpu_id.cpp | 33 ----------- library/cpp/yt/rseq/cpu_id-inl.h | 35 ------------ library/cpp/yt/rseq/cpu_id.cpp | 27 --------- library/cpp/yt/rseq/cpu_id.h | 28 ---------- library/cpp/yt/rseq/unittests/cpu_id_ut.cpp | 62 --------------------- library/cpp/yt/system/benchmarks/tscp.cpp | 9 --- library/cpp/yt/system/tscp-inl.h | 16 ------ library/cpp/yt/system/tscp.h | 10 ---- 9 files changed, 1 insertion(+), 221 deletions(-) delete mode 100644 library/cpp/yt/rseq/benchmarks/cpu_id.cpp delete mode 100644 library/cpp/yt/rseq/cpu_id-inl.h delete mode 100644 library/cpp/yt/rseq/cpu_id.cpp delete mode 100644 library/cpp/yt/rseq/cpu_id.h delete mode 100644 library/cpp/yt/rseq/unittests/cpu_id_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 1b12ffc7105..c0aa2716240 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -7c7cbc3873cc519ce596389242e391f8b327a4e4 +512e279df0984149e1e42eae439361aa6d4c6680 diff --git a/library/cpp/yt/rseq/benchmarks/cpu_id.cpp b/library/cpp/yt/rseq/benchmarks/cpu_id.cpp deleted file mode 100644 index 03842114f49..00000000000 --- a/library/cpp/yt/rseq/benchmarks/cpu_id.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include - -#include - -#include - -namespace NYT::NRseq { -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -// rseq: a single thread-local read. -void BM_GetCurrentCpuId(benchmark::State& state) -{ - for (auto _ : state) { - benchmark::DoNotOptimize(GetCurrentCpuId()); - } -} -BENCHMARK(BM_GetCurrentCpuId); - -// The fallback path: sched_getcpu() (vDSO getcpu). -void BM_SchedGetCpu(benchmark::State& state) -{ - for (auto _ : state) { - benchmark::DoNotOptimize(sched_getcpu()); - } -} -BENCHMARK(BM_SchedGetCpu); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NRseq diff --git a/library/cpp/yt/rseq/cpu_id-inl.h b/library/cpp/yt/rseq/cpu_id-inl.h deleted file mode 100644 index 5d31bdedcaf..00000000000 --- a/library/cpp/yt/rseq/cpu_id-inl.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef CPU_ID_INL_H_ -#error "Direct inclusion of this file is not allowed, include cpu_id.h" -// For the sake of sane code completion. -#include "cpu_id.h" -#endif - -#include "rseq_area.h" - -#include - -namespace NYT::NRseq { - -//////////////////////////////////////////////////////////////////////////////// - -namespace NDetail { - -int GetCurrentCpuIdSlow(); - -} // namespace NDetail - -Y_FORCE_INLINE int GetCurrentCpuId() -{ - // Branch-free read of the rseq cpu_id: the offset always points at a readable - // field. A data-dependent branch on the offset here would defeat load pipelining. - auto cpuId = NDetail::ReadRseqAreaField(offsetof(NDetail::TRseqArea, CpuId)); - // Negative means this thread is not registered yet or rseq is unavailable. - if (cpuId < 0) [[unlikely]] { - return NDetail::GetCurrentCpuIdSlow(); - } - return cpuId; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NRseq diff --git a/library/cpp/yt/rseq/cpu_id.cpp b/library/cpp/yt/rseq/cpu_id.cpp deleted file mode 100644 index c3ac3ff4796..00000000000 --- a/library/cpp/yt/rseq/cpu_id.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "cpu_id.h" -#include "rseq.h" -#include "rseq_area.h" - -#include - -#include - -namespace NYT::NRseq::NDetail { - -//////////////////////////////////////////////////////////////////////////////// - -YT_PREVENT_TLS_CACHING int GetCurrentCpuIdSlow() -{ - if (EnsureCurrentThreadRegistered()) { - auto cpuId = ReadRseqAreaField(offsetof(TRseqArea, CpuId)); - if (cpuId >= 0) { - return cpuId; - } - } - auto cpuId = ::sched_getcpu(); - return cpuId >= 0 ? cpuId : 0; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NRseq::NDetail diff --git a/library/cpp/yt/rseq/cpu_id.h b/library/cpp/yt/rseq/cpu_id.h deleted file mode 100644 index c41d25372f9..00000000000 --- a/library/cpp/yt/rseq/cpu_id.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -namespace NYT::NRseq { - -//////////////////////////////////////////////////////////////////////////////// - -/*! - * Index of the CPU the calling thread currently runs on, in [0, NumberOfCpus). - * The fast path is a single thread-local read of the kernel's rseq area; the first - * call per thread lazily registers rseq, and reads fall back to sched_getcpu() when - * rseq is unavailable. - * It is a hint (the thread may migrate right after) -- for sharding, not correctness. - * - * Fiber-TLS: the fast path is inlined and reads the thread pointer, so it must be - * reached only through a non-inlinable, fiber-switch-free frame (a virtual call or - * YT_PREVENT_TLS_CACHING; see library/cpp/yt/misc/tls.h). Otherwise the thread base - * may be cached and reused after the fiber resumes on another OS thread, reading a - * different -- possibly already-freed -- thread's area. - */ -int GetCurrentCpuId(); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT::NRseq - -#define CPU_ID_INL_H_ -#include "cpu_id-inl.h" -#undef CPU_ID_INL_H_ diff --git a/library/cpp/yt/rseq/unittests/cpu_id_ut.cpp b/library/cpp/yt/rseq/unittests/cpu_id_ut.cpp deleted file mode 100644 index 0d51a5d5bfd..00000000000 --- a/library/cpp/yt/rseq/unittests/cpu_id_ut.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include - -#include - -#include - -#include - -namespace NYT::NRseq { -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -std::vector GetAllowedCpus() -{ - cpu_set_t set; - CPU_ZERO(&set); - if (sched_getaffinity(0, sizeof(set), &set) != 0) { - return {}; - } - std::vector cpus; - for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { - if (CPU_ISSET(cpu, &set)) { - cpus.push_back(cpu); - } - } - return cpus; -} - -bool TryPinToCpu(int cpu) -{ - cpu_set_t set; - CPU_ZERO(&set); - CPU_SET(cpu, &set); - // Pinning the current thread to a single allowed CPU: the kernel migrates us - // onto it before sched_setaffinity returns, and keeps us there (it is the only - // permitted CPU), so there is no migration race afterwards. - return sched_setaffinity(0, sizeof(set), &set) == 0; -} - -//////////////////////////////////////////////////////////////////////////////// - -// When pinned to a CPU, GetCurrentCpuId must report exactly that CPU -- this checks -// the actual contract (the real running CPU), not just a range, and is independent -// of how many CPUs are online or how they are numbered. -TEST(TGetCurrentCpuIdTest, MatchesPinnedCpu) -{ - auto allowedCpus = GetAllowedCpus(); - if (allowedCpus.empty() || !TryPinToCpu(allowedCpus.front())) { - GTEST_SKIP() << "Cannot control CPU affinity in this environment"; - } - - for (int cpu : allowedCpus) { - ASSERT_TRUE(TryPinToCpu(cpu)); - EXPECT_EQ(GetCurrentCpuId(), cpu); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace -} // namespace NYT::NRseq diff --git a/library/cpp/yt/system/benchmarks/tscp.cpp b/library/cpp/yt/system/benchmarks/tscp.cpp index 067584a3198..fdbdfac80cc 100644 --- a/library/cpp/yt/system/benchmarks/tscp.cpp +++ b/library/cpp/yt/system/benchmarks/tscp.cpp @@ -16,15 +16,6 @@ void BM_TscpGet(benchmark::State& state) } BENCHMARK(BM_TscpGet); -// Cheaper, lower-precision counterpart: rseq fast path + non-serializing rdtsc. -void BM_TscpGetApproximate(benchmark::State& state) -{ - for (auto _ : state) { - benchmark::DoNotOptimize(TTscp::GetApproximate()); - } -} -BENCHMARK(BM_TscpGetApproximate); - //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/library/cpp/yt/system/tscp-inl.h b/library/cpp/yt/system/tscp-inl.h index b224deb84c5..2fefd81c8a6 100644 --- a/library/cpp/yt/system/tscp-inl.h +++ b/library/cpp/yt/system/tscp-inl.h @@ -4,10 +4,6 @@ #include "tscp.h" #endif -#if defined(__linux__) && (defined(__x86_64__) || defined(__aarch64__)) -#include -#endif - #include namespace NYT::NProfiling { @@ -60,18 +56,6 @@ inline TTscp TTscp::Get() }; } -inline TTscp TTscp::GetApproximate() -{ -#if defined(__linux__) && (defined(__x86_64__) || defined(__aarch64__)) - return TTscp{ - .Instant = GetApproximateCpuInstant(), - .ProcessorId = NRseq::GetCurrentCpuId() & (MaxProcessorId - 1), - }; -#else - return Get(); -#endif -} - //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NProfiling diff --git a/library/cpp/yt/system/tscp.h b/library/cpp/yt/system/tscp.h index 05e8f835719..fe20d5e43d6 100644 --- a/library/cpp/yt/system/tscp.h +++ b/library/cpp/yt/system/tscp.h @@ -25,16 +25,6 @@ struct TTscp //! Returns the current timestamp counter and processor id, both obtained from a single serializing rdtscp. static TTscp Get(); - - //! A cheaper, lower-precision counterpart of #Get: the processor id comes from the - //! rseq fast path (GetCurrentCpuId) and the instant from a non-serializing rdtsc - //! (GetApproximateCpuInstant), instead of a single serializing rdtscp. - //! Without rseq (anything but Linux x86-64/aarch64), same as #Get. - //! - //! WARNING (fiber TLS): inlined and reads the thread pointer (via GetCurrentCpuId), - //! so it must be reached only through a non-inlinable, fiber-switch-free frame (a - //! virtual call or YT_PREVENT_TLS_CACHING; see library/cpp/yt/misc/tls.h). - static TTscp GetApproximate(); }; //////////////////////////////////////////////////////////////////////////////// From a3c1bab1f7d254aa3618e649f74e8d76bf77eb26 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:07:10 +0000 Subject: [PATCH 03/17] Send RAW topic messages on executor threads (#58232) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + .../ydb-cpp-sdk/client/topic/write_session.h | 2 +- .../impl/write_session_impl.cpp | 44 ++++++++++++++-- .../impl/write_session_impl.h | 4 ++ .../persqueue_public/include/write_session.h | 2 +- .../ut/compress_executor_ut.cpp | 42 +++++++++++++++ src/client/topic/impl/write_session_impl.cpp | 52 +++++++++++++++++-- src/client/topic/impl/write_session_impl.h | 3 ++ .../topic/ut/topic_deferred_publish_ut.cpp | 2 - .../topic/ut/write_session_flush_ut.cpp | 47 +++++++++++++++++ 11 files changed, 189 insertions(+), 13 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index c0aa2716240..82e6e633b0b 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -512e279df0984149e1e42eae439361aa6d4c6680 +85f2d3d09b9f45b9e4fdbbbd0cdd85a141368192 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a585d9da8c..1a8cf7a098f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Topic and PersQueue writers now schedule RAW and pre-encoded messages on the configured compression executor, reducing request construction on the caller thread. + * Added optional `TReplicationDescription::GetTransactions()` to report assigned global-consistency replication transactions and versions pending transaction-ID assignment. * Added `TGetClusterStateSettings::IncludePileConnectivity()` and `TGetClusterStateResult::GetPileConnectivity()` for optional bridge static-node connectivity reports. diff --git a/include/ydb-cpp-sdk/client/topic/write_session.h b/include/ydb-cpp-sdk/client/topic/write_session.h index 7ad4ad1d992..d81e2087221 100644 --- a/include/ydb-cpp-sdk/client/topic/write_session.h +++ b/include/ydb-cpp-sdk/client/topic/write_session.h @@ -143,7 +143,7 @@ struct TWriteSessionSettings : public TRequestSettings { FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); - //! Executor for compression tasks. + //! Executor for compression tasks and sending messages. //! If not set, default executor will be used. FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor); diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index 49e8fb74ba8..c73cbf56bc6 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -377,6 +377,7 @@ void TWriteSessionImpl::WriteInternal( readyToAccept = TryIssueContinuationTokenImpl(); FlushWriteIfRequiredImpl(); } + PostSendTask(); if (readyToAccept) { EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); } @@ -1092,6 +1093,11 @@ size_t TWriteSessionImpl::WriteBatchImpl() { Y_ABORT_UNLESS(CurrentBatch.Messages.size() <= MaxBlockMessageCount); const bool skipCompression = Settings.Codec_ == ECodec::RAW || CurrentBatch.HasCodec(); + // Preserve ordering before adding a duplicate or decreasing manual sequence number. + if (SendImplScheduled && !OriginalMessagesToSend.empty() + && CurrentBatch.Messages.front().Id <= OriginalMessagesToSend.back().Id) { + SendImpl(); + } if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) { MessagesAcquired += static_cast(CurrentBatch.Acquire()); } @@ -1133,7 +1139,7 @@ size_t TWriteSessionImpl::WriteBatchImpl() { } CurrentBatch.Reset(); if (skipCompression) { - SendImpl(); + ScheduleSendImpl(); } return size; } @@ -1317,6 +1323,34 @@ void TWriteSessionImpl::UpdateTokenImpl(const NThreading::TFuture& Processor->Write(std::move(clientMessage)); } +void TWriteSessionImpl::ScheduleSendImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (SendImplScheduled) { + return; + } + + PendingSendTask = [cbContext = SelfContext] { + if (auto self = cbContext->LockShared()) { + std::lock_guard guard(self->Lock); + self->SendImplScheduled = false; + self->SendImpl(); + } + }; + SendImplScheduled = true; +} + +void TWriteSessionImpl::PostSendTask() { + IExecutor::TFunction task; + { + std::lock_guard guard(Lock); + task.swap(PendingSendTask); + } + if (task) { + CompressionExecutor->Post(std::move(task)); + } +} + void TWriteSessionImpl::SendImpl() { Y_ABORT_UNLESS(Lock.IsLocked()); @@ -1440,6 +1474,7 @@ void TWriteSessionImpl::HandleWakeUpImpl() { if (Aborting.load()) { return; } + SendImpl(); auto callback = [cbContext = SelfContext] (bool ok) { if (!ok) { @@ -1447,8 +1482,11 @@ void TWriteSessionImpl::HandleWakeUpImpl() { } if (auto self = cbContext->LockShared()) { - std::lock_guard guard(self->Lock); - self->HandleWakeUpImpl(); + { + std::lock_guard guard(self->Lock); + self->HandleWakeUpImpl(); + } + self->PostSendTask(); } }; diff --git a/src/client/persqueue_public/impl/write_session_impl.h b/src/client/persqueue_public/impl/write_session_impl.h index 061ad02656c..19e091107fb 100644 --- a/src/client/persqueue_public/impl/write_session_impl.h +++ b/src/client/persqueue_public/impl/write_session_impl.h @@ -365,6 +365,8 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, ui64 GetNextIdImpl(const std::optional& seqNo); ui64 GetSeqNoImpl(ui64 id); ui64 GetIdImpl(ui64 seqNo); + void ScheduleSendImpl(); + void PostSendTask(); void SendImpl(); void AbortImpl(); void CloseImpl(EStatus statusCode, NYdb::NIssue::TIssues&& issues); @@ -424,6 +426,8 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, const size_t MaxBlockMessageCount = 1; //!< Max message count that can be packed into a single block. In block version 0 is equal to 1 for compatibility bool Connected = false; bool Started = false; + bool SendImplScheduled = false; + IExecutor::TFunction PendingSendTask; std::atomic Aborting = 0; bool SessionEstablished = false; ui32 PartitionId = 0; diff --git a/src/client/persqueue_public/include/write_session.h b/src/client/persqueue_public/include/write_session.h index 1fdaeec414b..426d6f3d126 100644 --- a/src/client/persqueue_public/include/write_session.h +++ b/src/client/persqueue_public/include/write_session.h @@ -85,7 +85,7 @@ struct TWriteSessionSettings : public TRequestSettings { FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters); - //! Executor for compression tasks. + //! Executor for compression tasks and sending messages. //! If not set, default executor will be used. FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor); diff --git a/src/client/persqueue_public/ut/compress_executor_ut.cpp b/src/client/persqueue_public/ut/compress_executor_ut.cpp index 99cd373e27a..3ded10ae2e3 100644 --- a/src/client/persqueue_public/ut/compress_executor_ut.cpp +++ b/src/client/persqueue_public/ut/compress_executor_ut.cpp @@ -1,10 +1,52 @@ #include +#include +#include + +#include + #include namespace NYdb::NPersQueue::NTests { Y_UNIT_TEST_SUITE(CompressExecutor) { + Y_UNIT_TEST(RawWritesWithExternalInlineExecutor) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + auto pool = ::CreateThreadPool(1, 0); + auto executor = NAdapters::CreateExternalThreadPoolExecutorAdapter( + std::shared_ptr(pool.Release())); + + auto settings = setup.GetWriteSessionSettings(); + settings.Codec(ECodec::RAW).CompressionExecutor(executor); + auto writer = setup.GetPersQueueClient().CreateSimpleBlockingWriteSession(settings); + UNIT_ASSERT(writer->Write("message", std::nullopt, std::nullopt, TDuration::Seconds(30))); + UNIT_ASSERT(writer->Close(TDuration::Seconds(30))); + } + + Y_UNIT_TEST(RawWritesUseCompressionExecutor) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + for (const bool runExecutor : {false, true}) { + auto executor = NTopic::NTests::CreateThreadPoolManagedExecutor(1); + auto settings = setup.GetWriteSessionSettings(); + settings.Codec(ECodec::RAW) + .MessageGroupId(runExecutor ? "running-executor" : "held-executor") + .CompressionExecutor(executor); + auto writer = setup.GetPersQueueClient().CreateSimpleBlockingWriteSession(settings); + + for (const ui64 seqNo : {2, 2, 1, 3}) { + UNIT_ASSERT(writer->Write("message", seqNo, std::nullopt, TDuration::Seconds(30))); + } + UNIT_ASSERT_VALUES_EQUAL(executor->GetPlannedCount(), 1); + if (runExecutor) { + executor->RunAllTasks(); + } + UNIT_ASSERT(writer->Close(TDuration::Seconds(30))); + writer.reset(); + executor->RunAllTasks(); + executor->Stop(); + } + } + Y_UNIT_TEST(TestReorderedExecutor) { auto queue = std::make_shared>(); TYdbPqWriterTestHelper helper(TEST_CASE_NAME, queue); diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index 94142c07728..97349183652 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -652,7 +652,7 @@ NThreading::TFuture TWriteSessionImpl::WaitEvent() { } NThreading::TFuture TWriteSessionImpl::Flush() { - std::lock_guard guard(Lock); + std::unique_lock guard(Lock); if (Aborting.load()) { return NThreading::MakeFuture(false); } @@ -674,7 +674,10 @@ NThreading::TFuture TWriteSessionImpl::Flush() { message->InitFlushPromise(Connections); } - return message->FlushPromise.GetFuture(); + auto result = message->FlushPromise.GetFuture(); + guard.unlock(); + PostSendTask(); + return result; } void TWriteSessionImpl::TrySubscribeOnTransactionCommit(TTransactionBase* tx) @@ -856,6 +859,7 @@ void TWriteSessionImpl::WriteInternal(TContinuationToken&&, TWriteMessage&& mess readyToAccept = TryIssueContinuationTokenImpl(); FlushWriteIfRequiredImpl(); } + PostSendTask(); if (readyToAccept) { EventsQueue->PushEvent(TWriteSessionEvent::TReadyToAcceptEvent{IssueContinuationToken()}); } @@ -1771,6 +1775,11 @@ size_t TWriteSessionImpl::WriteBatchImpl() { } const bool skipCompression = Settings.Codec_ == ECodec::RAW || CurrentBatch.HasCodec(); + // Preserve ordering before adding a duplicate or decreasing manual sequence number. + if (!OriginalMessagesToSend.empty() + && CurrentBatch.Messages.front().Id <= OriginalMessagesToSend.back().Id) { + SendImpl(); + } if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) { MessagesAcquired += static_cast(CurrentBatch.Acquire()); } @@ -1833,7 +1842,7 @@ size_t TWriteSessionImpl::WriteBatchImpl() { } CurrentBatch.Reset(); if (skipCompression) { - SendImpl(); + ScheduleSendImpl(); } return size; } @@ -2048,6 +2057,34 @@ void TWriteSessionImpl::SendStandardBlock( } } +void TWriteSessionImpl::ScheduleSendImpl() { + Y_ABORT_UNLESS(Lock.IsLocked()); + + if (SendImplScheduled) { + return; + } + + PendingSendTask = [cbContext = SelfContext] { + if (auto self = cbContext->LockShared()) { + std::lock_guard guard(self->Lock); + self->SendImplScheduled = false; + self->SendImpl(); + } + }; + SendImplScheduled = true; +} + +void TWriteSessionImpl::PostSendTask() { + IExecutor::TFunction task; + { + std::lock_guard guard(Lock); + task.swap(PendingSendTask); + } + if (task) { + CompressionExecutor->Post(std::move(task)); + } +} + void TWriteSessionImpl::SendImpl() { Y_ABORT_UNLESS(Lock.IsLocked()); @@ -2123,6 +2160,7 @@ bool TWriteSessionImpl::Close(TDuration closeTimeout) { WriteBatchImpl(); } } + PostSendTask(); auto startTime = TInstant::Now(); auto remaining = closeTimeout; bool ready = false; @@ -2166,14 +2204,18 @@ void TWriteSessionImpl::HandleWakeUpImpl() { if (Aborting.load()) { return; } + SendImpl(); auto callback = [cbContext = SelfContext] (bool ok) { if (!ok) { return; } if (auto self = cbContext->LockShared()) { - std::lock_guard guard(self->Lock); - self->HandleWakeUpImpl(); + { + std::lock_guard guard(self->Lock); + self->HandleWakeUpImpl(); + } + self->PostSendTask(); } }; if (TInstant::Now() - LastTokenUpdate > UPDATE_TOKEN_PERIOD) { diff --git a/src/client/topic/impl/write_session_impl.h b/src/client/topic/impl/write_session_impl.h index c4185e7b94f..aa4e54cc93e 100644 --- a/src/client/topic/impl/write_session_impl.h +++ b/src/client/topic/impl/write_session_impl.h @@ -463,6 +463,8 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, uint64_t GetNextIdImpl(const std::optional& seqNo); uint64_t GetSeqNoImpl(uint64_t id); uint64_t GetIdImpl(uint64_t seqNo); + void ScheduleSendImpl(); + void PostSendTask(); void SendImpl(); void SendBatchBlock(const TBlock& block, Ydb::Topic::StreamWriteMessage_WriteRequest* writeRequest); void SendStandardBlock(const TBlock& block, Ydb::Topic::StreamWriteMessage_WriteRequest* writeRequest); @@ -538,6 +540,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, bool Connected = false; bool Started = false; std::atomic SendImplScheduled = false; + IExecutor::TFunction PendingSendTask; std::atomic Aborting = 0; bool SessionEstablished = false; bool BatchingSupported = false; diff --git a/src/client/topic/ut/topic_deferred_publish_ut.cpp b/src/client/topic/ut/topic_deferred_publish_ut.cpp index 6fdb27238ae..363429fe7a9 100644 --- a/src/client/topic/ut/topic_deferred_publish_ut.cpp +++ b/src/client/topic/ut/topic_deferred_publish_ut.cpp @@ -77,8 +77,6 @@ class TDeferredWriteHelper { { Settings_.Path(topicPath); Settings_.ProducerId("deferred-producer"); - // RAW: sync SendImpl per write so consecutive Writes are not coalesced into one - // write_request (GZIP async compress can batch them and hit unsorted seqNo). Settings_.Codec(ECodec::RAW); Session_ = Client_.CreateWriteSession(Settings_); } diff --git a/src/client/topic/ut/write_session_flush_ut.cpp b/src/client/topic/ut/write_session_flush_ut.cpp index 53526b938ff..f6605055796 100644 --- a/src/client/topic/ut/write_session_flush_ut.cpp +++ b/src/client/topic/ut/write_session_flush_ut.cpp @@ -1,8 +1,12 @@ #include "ut_utils/topic_sdk_test_setup.h" +#include #include +#include + #include +#include namespace NYdb::inline V3::NTopic::NTests { namespace { @@ -24,6 +28,49 @@ TContinuationToken WaitForWriteToken(IWriteSession& session) { } // anonymous namespace Y_UNIT_TEST_SUITE(WriteSessionFlush) { + Y_UNIT_TEST(RawWritesWithExternalInlineExecutor) { + TTopicSdkTestSetup setup(TEST_CASE_NAME); + auto client = setup.MakeClient(); + auto pool = ::CreateThreadPool(1, 0); + auto executor = NAdapters::CreateExternalThreadPoolExecutorAdapter( + std::shared_ptr(pool.Release())); + + auto session = client.CreateSimpleBlockingWriteSession( + TWriteSessionSettings() + .Path(setup.GetTopicPath()) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .Codec(ECodec::RAW) + .CompressionExecutor(executor)); + UNIT_ASSERT(session->Write("message", std::nullopt, std::nullopt, TDuration::Seconds(30))); + UNIT_ASSERT(session->Close(TDuration::Seconds(30))); + } + + Y_UNIT_TEST(RawWritesUseCompressionExecutor) { + TTopicSdkTestSetup setup(TEST_CASE_NAME); + auto client = setup.MakeClient(); + for (const bool runExecutor : {false, true}) { + auto executor = CreateThreadPoolManagedExecutor(1); + auto session = client.CreateSimpleBlockingWriteSession( + TWriteSessionSettings() + .Path(setup.GetTopicPath()) + .MessageGroupId(runExecutor ? "running-executor" : "held-executor") + .Codec(ECodec::RAW) + .CompressionExecutor(executor)); + + for (const uint64_t seqNo : {2, 2, 1, 3}) { + UNIT_ASSERT(session->Write("message", seqNo, std::nullopt, TDuration::Seconds(30))); + } + UNIT_ASSERT_VALUES_EQUAL(executor->GetPlannedCount(), 1); + if (runExecutor) { + executor->RunAllTasks(); + } + UNIT_ASSERT(session->Close(TDuration::Seconds(30))); + session.reset(); + executor->RunAllTasks(); + executor->Stop(); + } + } + Y_UNIT_TEST(CloseImmediatelyAfterFlush) { TTopicSdkTestSetup setup(TEST_CASE_NAME); auto client = setup.MakeClient(); From 6b8ff6fecd6a8b95e1e5cea6d48dbbff5385755f Mon Sep 17 00:00:00 2001 From: Sergey M Date: Wed, 11 Nov 2026 17:07:18 +0000 Subject: [PATCH 04/17] Add preparatory HNSW API and schema definitions (#57005) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/table/table.h | 35 +++ include/ydb-cpp-sdk/client/table/table_enum.h | 1 + src/api/protos/ydb_table.proto | 18 ++ src/client/table/out.cpp | 4 + src/client/table/table.cpp | 106 ++++++- tests/unit/client/table/table_ut.cpp | 273 ++++++++++++++++++ 7 files changed, 436 insertions(+), 3 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 82e6e633b0b..27d78e43b36 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -85f2d3d09b9f45b9e4fdbbbd0cdd85a141368192 +9942f098c3f9bc976c56d2998d4206f1427957aa diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 321a7e1726d..4ab52439c92 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -324,6 +324,11 @@ struct TGlobalIndexSettings { struct TVectorIndexSettings { public: + static constexpr uint64_t DefaultMinRows = 10000; + static constexpr uint32_t DefaultM = 16; + static constexpr uint32_t DefaultEfConstruction = 200; + static constexpr uint64_t DefaultDeltaRows = 10000; + enum class EMetric { Unspecified = 0, InnerProduct, @@ -350,6 +355,11 @@ struct TVectorIndexSettings { EMetric Metric = EMetric::Unspecified; EVectorType VectorType = EVectorType::Unspecified; uint32_t VectorDimension = 0; + uint64_t MinRows = DefaultMinRows; + uint32_t M = DefaultM; + uint32_t EfConstruction = DefaultEfConstruction; + // Rebuild after more than this many distinct rows change since the graph snapshot. + uint64_t DeltaRows = DefaultDeltaRows; static TVectorIndexSettings FromProto(const Ydb::Table::VectorIndexSettings& proto); @@ -400,6 +410,15 @@ struct TKMeansTreeSettings { void Out(IOutputStream &o) const; }; +struct THnswIndexSettings { + TKMeansTreeSettings VectorSettings; + std::vector DataColumns; + TGlobalIndexSettings LevelTableSettings; + TGlobalIndexSettings PostingTableSettings; + // Used only by CreatePrefixedHnswIndex. + TGlobalIndexSettings PrefixTableSettings; +}; + struct TFulltextIndexSettings { public: enum class ETokenizer { @@ -533,6 +552,18 @@ class TIndexDescription { const TGlobalIndexSettings& prefixTableSettings = {} ); + static TIndexDescription CreateHnswIndex( + const std::string& name, + const std::string& vectorColumn, + const THnswIndexSettings& settings + ); + + static TIndexDescription CreatePrefixedHnswIndex( + const std::string& name, + const std::vector& indexColumns, + const THnswIndexSettings& settings + ); + static TIndexDescription CreateFulltextPlainIndex( const std::string& name, const std::vector& indexColumns, @@ -1200,6 +1231,8 @@ class TTableDescription { // vector KMeansTree void AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings); void AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings); + void AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings); + void AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings); // fulltext void AddFulltextIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const TFulltextIndexSettings& indexSettings); void AddFulltextIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns, const TFulltextIndexSettings& indexSettings); @@ -1457,6 +1490,8 @@ class TTableBuilder { // vector KMeansTree TTableBuilder& AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings); TTableBuilder& AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings); + TTableBuilder& AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings); + TTableBuilder& AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings); // fulltext TTableBuilder& AddFulltextIndex(const std::string& indexName, const std::vector& indexColumns, const TFulltextIndexSettings& indexSettings); diff --git a/include/ydb-cpp-sdk/client/table/table_enum.h b/include/ydb-cpp-sdk/client/table/table_enum.h index bc195a48022..99d468f072b 100644 --- a/include/ydb-cpp-sdk/client/table/table_enum.h +++ b/include/ydb-cpp-sdk/client/table/table_enum.h @@ -41,6 +41,7 @@ enum class EIndexType { LocalBloomFilter, LocalBloomNgramFilter, LocalMinMax, + GlobalHnsw, Unknown = std::numeric_limits::max() }; diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index a705b884240..a6c7aec1bad 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -96,6 +96,18 @@ message VectorIndexSettings { optional VectorType vector_type = 2; optional uint32 vector_dimension = 3; + + // HNSW is used only for partitions with at least this many rows. Smaller + // partitions are searched by brute force. The default is 10000 rows. + optional uint64 min_rows = 4; + + // Graph construction parameters. Defaults: M=16, ef_construction=200. + optional uint32 m = 5; + optional uint32 ef_construction = 6; + // Rebuild when the number of distinct rows with committed vector changes + // since the graph snapshot exceeds this absolute count. The default is 10000. + // Zero requests rebuilding after any changed row. + optional uint64 delta_rows = 7; } message KMeansTreeSettings { @@ -419,6 +431,9 @@ message TableIndex { LocalBloomNgramFilterIndex local_bloom_ngram_filter_index = 11; GlobalJsonIndex global_json_index = 12; LocalMinMaxIndex local_min_max_index = 14; + // Intentionally reuses the k-means message for the preparatory HNSW API. + // Its level/posting/prefix table settings are placeholders for HNSW impl tables. + GlobalVectorKMeansTreeIndex global_hnsw_index = 15; } // list of columns content to be copied in to index table repeated string data_columns = 5; @@ -451,6 +466,9 @@ message TableIndexDescription { LocalBloomNgramFilterIndex local_bloom_ngram_filter_index = 13; GlobalJsonIndex global_json_index = 14; LocalMinMaxIndex local_min_max_index = 15; + // Intentionally mirrors TableIndex.global_hnsw_index, including its + // placeholder level/posting/prefix table settings for HNSW impl tables. + GlobalVectorKMeansTreeIndex global_hnsw_index = 16; } Status status = 4; // list of columns content to be copied in to index table diff --git a/src/client/table/out.cpp b/src/client/table/out.cpp index 7f42807414a..672e5c9aefc 100644 --- a/src/client/table/out.cpp +++ b/src/client/table/out.cpp @@ -85,6 +85,10 @@ Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings, stream, value) { "{ " << value.Metric << ", vector_type: " << value.VectorType << ", vector_dimension: " << value.VectorDimension << + ", min_rows: " << value.MinRows << + ", M: " << value.M << + ", ef_construction: " << value.EfConstruction << + ", delta_rows: " << value.DeltaRows << " }"; } diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 559f8688096..568f8f4e7b1 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -915,6 +915,14 @@ void TTableDescription::AddVectorKMeansTreeIndex(const std::string& indexName, c Impl_->AddVectorKMeansTreeIndex(indexName, EIndexType::GlobalVectorKMeansTree, indexColumns, dataColumns, indexSettings); } +void TTableDescription::AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings) { + Impl_->AddVectorKMeansTreeIndex(indexName, EIndexType::GlobalHnsw, indexColumns, indexSettings); +} + +void TTableDescription::AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings) { + Impl_->AddVectorKMeansTreeIndex(indexName, EIndexType::GlobalHnsw, indexColumns, dataColumns, indexSettings); +} + void TTableDescription::AddFulltextIndex(const std::string& indexName, EIndexType indexType, const std::vector& indexColumns, const TFulltextIndexSettings& indexSettings) { Impl_->AddFulltextIndex(indexName, indexType, indexColumns, indexSettings); } @@ -1441,6 +1449,16 @@ TTableBuilder& TTableBuilder::AddVectorKMeansTreeIndex(const std::string& indexN return *this; } +TTableBuilder& TTableBuilder::AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TKMeansTreeSettings& indexSettings) { + TableDescription_.AddHnswIndex(indexName, indexColumns, dataColumns, indexSettings); + return *this; +} + +TTableBuilder& TTableBuilder::AddHnswIndex(const std::string& indexName, const std::vector& indexColumns, const TKMeansTreeSettings& indexSettings) { + TableDescription_.AddHnswIndex(indexName, indexColumns, indexSettings); + return *this; +} + TTableBuilder& TTableBuilder::AddFulltextIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TFulltextIndexSettings& indexSettings) { TableDescription_.AddFulltextIndex(indexName, EIndexType::GlobalFulltextPlain, indexColumns, dataColumns, indexSettings); return *this; @@ -2717,6 +2735,28 @@ TIndexDescription TIndexDescription::CreatePrefixedVectorIndex( ); } +TIndexDescription TIndexDescription::CreateHnswIndex( + const std::string& name, + const std::string& vectorColumn, + const THnswIndexSettings& settings +) { + return TIndexDescription( + name, EIndexType::GlobalHnsw, {vectorColumn}, settings.DataColumns, + {settings.LevelTableSettings, settings.PostingTableSettings}, settings.VectorSettings + ); +} + +TIndexDescription TIndexDescription::CreatePrefixedHnswIndex( + const std::string& name, + const std::vector& indexColumns, + const THnswIndexSettings& settings +) { + return TIndexDescription( + name, EIndexType::GlobalHnsw, indexColumns, settings.DataColumns, + {settings.LevelTableSettings, settings.PostingTableSettings, settings.PrefixTableSettings}, settings.VectorSettings + ); +} + TIndexDescription TIndexDescription::CreateFulltextPlainIndex( const std::string& name, const std::vector& indexColumns, @@ -2925,6 +2965,10 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const Ydb::Table::VectorInd .Metric = covertMetric(), .VectorType = convertVectorType(), .VectorDimension = proto.vector_dimension(), + .MinRows = proto.has_min_rows() ? proto.min_rows() : DefaultMinRows, + .M = proto.has_m() ? proto.m() : DefaultM, + .EfConstruction = proto.has_ef_construction() ? proto.ef_construction() : DefaultEfConstruction, + .DeltaRows = proto.has_delta_rows() ? proto.delta_rows() : DefaultDeltaRows, }; } @@ -2976,6 +3020,26 @@ void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings settings.set_metric(convertMetric()); settings.set_vector_type(convertVectorType()); settings.set_vector_dimension(VectorDimension); + if (MinRows != DefaultMinRows) { + settings.set_min_rows(MinRows); + } else { + settings.clear_min_rows(); + } + if (M != DefaultM) { + settings.set_m(M); + } else { + settings.clear_m(); + } + if (EfConstruction != DefaultEfConstruction) { + settings.set_ef_construction(EfConstruction); + } else { + settings.clear_ef_construction(); + } + if (DeltaRows != DefaultDeltaRows) { + settings.set_delta_rows(DeltaRows); + } else { + settings.clear_delta_rows(); + } } void TVectorIndexSettings::Out(IOutputStream& o) const { @@ -3283,6 +3347,22 @@ TIndexDescription TIndexDescription::FromProto(const TProto& proto) { specializedIndexSettings = TKMeansTreeSettings::FromProto(vectorProto.vector_settings()); break; } + case TProto::kGlobalHnswIndex: { + type = EIndexType::GlobalHnsw; + const auto& vectorProto = proto.global_hnsw_index(); + const bool prefixVectorIndex = indexColumns.size() > 1; + globalIndexSettings.resize(prefixVectorIndex ? 3 : 2); + globalIndexSettings[TGlobalIndexSettings::VectorKMeansTreeLevelTablePosition] = + TGlobalIndexSettings::FromProto(vectorProto.level_table_settings()); + globalIndexSettings[TGlobalIndexSettings::VectorKMeansTreePostingTablePosition] = + TGlobalIndexSettings::FromProto(vectorProto.posting_table_settings()); + if (prefixVectorIndex) { + globalIndexSettings[TGlobalIndexSettings::VectorKMeansTreePrefixTablePosition] = + TGlobalIndexSettings::FromProto(vectorProto.prefix_table_settings()); + } + specializedIndexSettings = TKMeansTreeSettings::FromProto(vectorProto.vector_settings()); + break; + } case TProto::kGlobalFulltextPlainIndex: { type = EIndexType::GlobalFulltextPlain; const auto& fulltextProto = proto.global_fulltext_plain_index(); @@ -3367,8 +3447,11 @@ void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { GlobalIndexSettings_.at(0).SerializeTo(settings); break; } - case EIndexType::GlobalVectorKMeansTree: { - auto* global_vector_kmeans_tree_index = proto.mutable_global_vector_kmeans_tree_index(); + case EIndexType::GlobalVectorKMeansTree: + case EIndexType::GlobalHnsw: { + auto* global_vector_kmeans_tree_index = IndexType_ == EIndexType::GlobalHnsw + ? proto.mutable_global_hnsw_index() + : proto.mutable_global_vector_kmeans_tree_index(); auto& level_settings = *global_vector_kmeans_tree_index->mutable_level_table_settings(); auto& posting_settings = *global_vector_kmeans_tree_index->mutable_posting_table_settings(); auto& vector_settings = *global_vector_kmeans_tree_index->mutable_vector_settings(); @@ -3383,6 +3466,24 @@ void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { } if (const auto* settings = std::get_if(&SpecializedIndexSettings_)) { settings->SerializeTo(vector_settings); + if (IndexType_ == EIndexType::GlobalHnsw) { + // A full HNSW definition must retain explicit defaults when + // merged over an existing index definition during rebuild. + auto* vector = vector_settings.mutable_settings(); + vector->set_min_rows(settings->Settings.MinRows); + vector->set_m(settings->Settings.M); + vector->set_ef_construction(settings->Settings.EfConstruction); + vector->set_delta_rows(settings->Settings.DeltaRows); + } + } + if (IndexType_ == EIndexType::GlobalVectorKMeansTree && vector_settings.has_settings()) { + // Older servers reject unknown vector settings. These parameters + // apply only to HNSW and must not change existing k-means requests. + auto* settings = vector_settings.mutable_settings(); + settings->clear_min_rows(); + settings->clear_m(); + settings->clear_ef_construction(); + settings->clear_delta_rows(); } break; } @@ -3473,6 +3574,7 @@ void TIndexDescription::Out(IOutputStream& o) const { case EIndexType::Unknown: break; case EIndexType::GlobalVectorKMeansTree: + case EIndexType::GlobalHnsw: if (auto settings = std::get_if(&SpecializedIndexSettings_)) { o << ", vector_settings: " << *settings; } diff --git a/tests/unit/client/table/table_ut.cpp b/tests/unit/client/table/table_ut.cpp index 1baebc4d565..719027032af 100644 --- a/tests/unit/client/table/table_ut.cpp +++ b/tests/unit/client/table/table_ut.cpp @@ -19,8 +19,10 @@ #include #include #include +#include #include #include +#include using namespace NYdb; @@ -1089,3 +1091,274 @@ TEST(VectorIndexSettings, Float8RoundTrip) { EXPECT_EQ(restoredTree.Levels, 3); } } + +TEST(VectorIndexSettings, IndexTypeNumericCompatibility) { + using NTable::EIndexType; + // Index types are public API: adding HNSW must preserve existing values. + const std::pair existing[] = { + {EIndexType::GlobalSync, 0}, + {EIndexType::GlobalAsync, 1}, + {EIndexType::GlobalUnique, 2}, + {EIndexType::GlobalVectorKMeansTree, 3}, + {EIndexType::GlobalFulltextPlain, 4}, + {EIndexType::GlobalFulltextRelevance, 5}, + {EIndexType::GlobalJson, 6}, + {EIndexType::LocalBloomFilter, 7}, + {EIndexType::LocalBloomNgramFilter, 8}, + {EIndexType::LocalMinMax, 9}, + }; + for (const auto& [type, value] : existing) { + EXPECT_EQ(static_cast(type), value); + } + EXPECT_EQ(static_cast(EIndexType::GlobalHnsw), 10); + EXPECT_EQ(static_cast(EIndexType::Unknown), std::numeric_limits::max()); +} + +TEST(VectorIndexSettings, HnswDefaultsAndAbsoluteDeltaRows) { + Ydb::Table::VectorIndexSettings proto; + const auto defaults = NTable::TVectorIndexSettings::FromProto(proto); + EXPECT_EQ(defaults.MinRows, 10000u); + EXPECT_EQ(defaults.M, 16u); + EXPECT_EQ(defaults.EfConstruction, 200u); + EXPECT_EQ(defaults.DeltaRows, 10000u); + auto settings = defaults; + settings.MinRows = 0; + settings.M = 24; + settings.EfConstruction = 400; + settings.DeltaRows = 1ULL << 40; + settings.SerializeTo(proto); + EXPECT_TRUE(proto.has_min_rows()); + EXPECT_EQ(proto.min_rows(), 0u); + EXPECT_EQ(proto.m(), 24u); + EXPECT_EQ(proto.ef_construction(), 400u); + EXPECT_EQ(proto.delta_rows(), 1ULL << 40); + const auto restored = NTable::TVectorIndexSettings::FromProto(proto); + EXPECT_EQ(restored.MinRows, settings.MinRows); + EXPECT_EQ(restored.M, settings.M); + EXPECT_EQ(restored.EfConstruction, settings.EfConstruction); + EXPECT_EQ(restored.DeltaRows, settings.DeltaRows); + settings.DeltaRows = 0; + proto.Clear(); + settings.SerializeTo(proto); + EXPECT_TRUE(proto.has_delta_rows()); + EXPECT_EQ(NTable::TVectorIndexSettings::FromProto(proto).DeltaRows, 0u); +} + +TEST(VectorIndexSettings, HnswWireFieldNumbers) { + Ydb::Table::VectorIndexSettings proto; + // min_rows/M/ef_construction/delta_rows use tags 4/5/6/7. + // Unknown tag 8 does not affect the supported settings. + ASSERT_TRUE(proto.ParseFromString(std::string("\x20\x01\x28\x18\x30\xc8\x01\x38\x32\x40\x05", 11))); + const auto settings = NTable::TVectorIndexSettings::FromProto(proto); + EXPECT_EQ(settings.MinRows, 1u); + EXPECT_EQ(settings.M, 24u); + EXPECT_EQ(settings.EfConstruction, 200u); + EXPECT_EQ(settings.DeltaRows, 50u); + EXPECT_TRUE(proto.has_delta_rows()); +} + +TEST(VectorIndexSettings, HnswIndexFactoriesAndProtoRoundTrip) { + for (const bool prefixed : {false, true}) { + Ydb::Table::KMeansTreeSettings tree; + tree.set_levels(1); + tree.set_clusters(1); + auto* vector = tree.mutable_settings(); + vector->set_metric(Ydb::Table::VectorIndexSettings::SIMILARITY_INNER_PRODUCT); + vector->set_vector_type(Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT); + vector->set_vector_dimension(200); + vector->set_min_rows(0); + vector->set_m(24); + vector->set_ef_construction(300); + vector->set_delta_rows(1ULL << 40); + NTable::THnswIndexSettings settings; + settings.VectorSettings = NTable::TKMeansTreeSettings::FromProto(tree); + settings.DataColumns = {"payload"}; + settings.LevelTableSettings.Partitions = uint64_t{2}; + settings.PostingTableSettings.Partitions = uint64_t{3}; + settings.PrefixTableSettings.Partitions = uint64_t{4}; + const std::vector indexColumns = prefixed + ? std::vector{"scope", "emb"} + : std::vector{"emb"}; + const auto factory = prefixed + ? NTable::TIndexDescription::CreatePrefixedHnswIndex("vidx", {"scope", "emb"}, settings) + : NTable::TIndexDescription::CreateHnswIndex("vidx", "emb", settings); + Ydb::Table::TableIndex request; + factory.SerializeTo(request); + ASSERT_EQ(request.type_case(), Ydb::Table::TableIndex::kGlobalHnswIndex); + EXPECT_EQ(request.index_columns_size(), prefixed ? 2 : 1); + + auto check = [&](const NTable::TIndexDescription& index) { + EXPECT_EQ(index.GetIndexName(), "vidx"); + EXPECT_EQ(index.GetIndexType(), NTable::EIndexType::GlobalHnsw); + EXPECT_EQ(index.GetIndexColumns(), indexColumns); + EXPECT_EQ(index.GetDataColumns(), settings.DataColumns); + Ydb::Table::TableIndex serialized; + index.SerializeTo(serialized); + ASSERT_EQ(serialized.type_case(), Ydb::Table::TableIndex::kGlobalHnswIndex); + const auto& hnsw = serialized.global_hnsw_index(); + EXPECT_EQ(hnsw.level_table_settings().uniform_partitions(), 2u); + EXPECT_EQ(hnsw.posting_table_settings().uniform_partitions(), 3u); + EXPECT_EQ(hnsw.has_prefix_table_settings(), prefixed); + if (prefixed) { + EXPECT_EQ(hnsw.prefix_table_settings().uniform_partitions(), 4u); + } + const auto& actual = hnsw.vector_settings(); + EXPECT_EQ(actual.levels(), 1u); + EXPECT_EQ(actual.clusters(), 1u); + EXPECT_EQ(actual.settings().SerializeAsString(), vector->SerializeAsString()); + }; + check(factory); + check(TProtoAccessor::FromProto(request)); + Ydb::Table::TableIndexDescription description; + description.set_name(request.name()); + *description.mutable_index_columns() = request.index_columns(); + *description.mutable_data_columns() = request.data_columns(); + *description.mutable_global_hnsw_index() = request.global_hnsw_index(); + check(TProtoAccessor::FromProto(description)); + } +} + +TEST(VectorIndexSettings, LegacyMissingIndexKindIsPreserved) { + Ydb::Table::TableIndexDescription description; + description.set_name("legacy_index"); + EXPECT_EQ(TProtoAccessor::FromProto(description).GetIndexType(), NTable::EIndexType::GlobalSync); +} + +TEST(VectorIndexSettings, KMeansIndexKeepsLegacyWireSettings) { + NTable::TKMeansTreeSettings settings; + settings.Settings.Metric = NTable::TVectorIndexSettings::EMetric::InnerProduct; + settings.Settings.VectorType = NTable::TVectorIndexSettings::EVectorType::Float; + settings.Settings.VectorDimension = 200; + settings.Settings.MinRows = 0; + settings.Settings.M = 24; + settings.Settings.EfConstruction = 300; + settings.Settings.DeltaRows = 0; + settings.Levels = 1; + settings.Clusters = 2; + const auto index = NTable::TIndexDescription::CreateVectorIndex("vidx", "emb", settings); + Ydb::Table::TableIndex request; + index.SerializeTo(request); + ASSERT_EQ(request.type_case(), Ydb::Table::TableIndex::kGlobalVectorKmeansTreeIndex); + const auto& vector = request.global_vector_kmeans_tree_index().vector_settings().settings(); + EXPECT_FALSE(vector.has_min_rows()); + EXPECT_FALSE(vector.has_m()); + EXPECT_FALSE(vector.has_ef_construction()); + EXPECT_FALSE(vector.has_delta_rows()); + EXPECT_EQ(vector.vector_dimension(), 200u); + + request.Clear(); + NTable::TIndexDescription("unconfigured", NTable::EIndexType::GlobalVectorKMeansTree, {"emb"}) + .SerializeTo(request); + EXPECT_FALSE(request.global_vector_kmeans_tree_index().vector_settings().has_settings()); +} + +TEST(VectorIndexSettings, DefaultHnswSettingsClearPreviousOverrides) { + NTable::TVectorIndexSettings settings; + settings.MinRows = 0; + settings.M = 24; + settings.EfConstruction = 300; + settings.DeltaRows = 0; + Ydb::Table::VectorIndexSettings proto; + settings.SerializeTo(proto); + ASSERT_TRUE(proto.has_min_rows()); + ASSERT_TRUE(proto.has_m()); + ASSERT_TRUE(proto.has_ef_construction()); + ASSERT_TRUE(proto.has_delta_rows()); + + NTable::TVectorIndexSettings().SerializeTo(proto); + EXPECT_FALSE(proto.has_min_rows()); + EXPECT_FALSE(proto.has_m()); + EXPECT_FALSE(proto.has_ef_construction()); + EXPECT_FALSE(proto.has_delta_rows()); + const auto restored = NTable::TVectorIndexSettings::FromProto(proto); + EXPECT_EQ(restored.MinRows, 10000u); + EXPECT_EQ(restored.M, 16u); + EXPECT_EQ(restored.EfConstruction, 200u); + EXPECT_EQ(restored.DeltaRows, 10000u); +} + +TEST(VectorIndexSettings, HnswIndexDefaultsHavePresence) { + NTable::TKMeansTreeSettings settings; + settings.Settings.Metric = NTable::TVectorIndexSettings::EMetric::InnerProduct; + settings.Settings.VectorType = NTable::TVectorIndexSettings::EVectorType::Float; + settings.Settings.VectorDimension = 200; + settings.Levels = 1; + settings.Clusters = 1; + const auto index = NTable::TIndexDescription::CreateHnswIndex("vidx", "emb", {.VectorSettings = settings}); + Ydb::Table::TableIndex request; + index.SerializeTo(request); + const auto& vector = request.global_hnsw_index().vector_settings().settings(); + ASSERT_TRUE(vector.has_min_rows()); + ASSERT_TRUE(vector.has_m()); + ASSERT_TRUE(vector.has_ef_construction()); + ASSERT_TRUE(vector.has_delta_rows()); + EXPECT_EQ(vector.min_rows(), 10000u); + EXPECT_EQ(vector.m(), 16u); + EXPECT_EQ(vector.ef_construction(), 200u); + EXPECT_EQ(vector.delta_rows(), 10000u); +} + +TEST(VectorIndexSettings, HnswTableBuilderPreservesColumnsAndSettings) { + NTable::TKMeansTreeSettings settings; + settings.Settings.Metric = NTable::TVectorIndexSettings::EMetric::InnerProduct; + settings.Settings.VectorType = NTable::TVectorIndexSettings::EVectorType::Float; + settings.Settings.VectorDimension = 200; + settings.Settings.MinRows = 0; + settings.Settings.M = 24; + settings.Settings.EfConstruction = 300; + settings.Settings.DeltaRows = 1ULL << 40; + settings.Levels = 1; + settings.Clusters = 1; + + const auto table = NTable::TTableBuilder() + .AddHnswIndex("plain", {"emb"}, settings) + .AddHnswIndex("covering", {"scope", "emb"}, {"payload"}, settings) + .Build(); + const auto indexes = table.GetIndexDescriptions(); + ASSERT_EQ(indexes.size(), 2u); + + Ydb::Table::KMeansTreeSettings expectedSettings; + settings.SerializeTo(expectedSettings); + for (size_t i = 0; i < indexes.size(); ++i) { + const std::string name = i == 0 ? "plain" : "covering"; + const std::vector columns = i == 0 + ? std::vector{"emb"} + : std::vector{"scope", "emb"}; + const std::vector dataColumns = i == 0 + ? std::vector{} + : std::vector{"payload"}; + EXPECT_EQ(indexes[i].GetIndexName(), name); + EXPECT_EQ(indexes[i].GetIndexType(), NTable::EIndexType::GlobalHnsw); + EXPECT_EQ(indexes[i].GetIndexColumns(), columns); + EXPECT_EQ(indexes[i].GetDataColumns(), dataColumns); + + Ydb::Table::TableIndex request; + indexes[i].SerializeTo(request); + EXPECT_EQ(request.name(), name); + EXPECT_EQ(std::vector(request.index_columns().begin(), request.index_columns().end()), columns); + EXPECT_EQ(std::vector(request.data_columns().begin(), request.data_columns().end()), dataColumns); + ASSERT_EQ(request.type_case(), Ydb::Table::TableIndex::kGlobalHnswIndex); + EXPECT_EQ(request.global_hnsw_index().vector_settings().SerializeAsString(), expectedSettings.SerializeAsString()); + } +} + +TEST(VectorIndexSettings, HnswDiagnosticOutputIncludesSettings) { + NTable::TKMeansTreeSettings settings; + settings.Settings.Metric = NTable::TVectorIndexSettings::EMetric::InnerProduct; + settings.Settings.VectorType = NTable::TVectorIndexSettings::EVectorType::Float; + settings.Settings.VectorDimension = 200; + settings.Settings.MinRows = 0; + settings.Settings.M = 24; + settings.Settings.EfConstruction = 300; + settings.Settings.DeltaRows = 1ULL << 40; + + const TString expected = "{ similarity: inner_product, vector_type: float, vector_dimension: 200, " + "min_rows: 0, M: 24, ef_construction: 300, delta_rows: 1099511627776 }"; + EXPECT_EQ(ToString(settings.Settings), expected); + const auto index = NTable::TIndexDescription::CreateHnswIndex("vidx", "emb", {.VectorSettings = settings}); + const auto output = index.ToString(); + EXPECT_NE(output.find("vector_settings: { settings: " + std::string(expected)), std::string::npos); + + const NTable::TIndexDescription unconfigured("vidx", NTable::EIndexType::GlobalHnsw, {"emb"}); + EXPECT_EQ(unconfigured.ToString().find("vector_settings:"), std::string::npos); +} From 85aa2db2cc780eb2b1b6f9ba33ab03cc4f0b2d25 Mon Sep 17 00:00:00 2001 From: YDBot Date: Wed, 11 Nov 2026 17:07:25 +0000 Subject: [PATCH 05/17] Merge pull request #58709 from ydb-platform/robots/branch-sync/merge-rightlib-261109-0844 --- .github/last_commit.txt | 2 +- .../monlib/encode/spack/spack_v1_decoder.cpp | 12 +- .../cpp/monlib/encode/spack/spack_v1_ut.cpp | 61 +++++++++ library/cpp/resource/registry.cpp | 95 ++++++++++---- library/cpp/resource/static_blob_default.cpp | 5 + library/cpp/resource/static_hash.h | 122 ++++++++++++++++++ 6 files changed, 267 insertions(+), 30 deletions(-) create mode 100644 library/cpp/resource/static_blob_default.cpp create mode 100644 library/cpp/resource/static_hash.h diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 27d78e43b36..98b7d36f7d9 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -9942f098c3f9bc976c56d2998d4206f1427957aa +a5253b0ae0647e2650b30b7042bec226628a187f diff --git a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp index a8f0d7bbf41..96cb7b380ac 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_decoder.cpp @@ -67,9 +67,13 @@ namespace NMonitoring { TVector valuesBuf; if (Header_.Version == SV1_03 || Header_.Version == SV1_04) { - auto namesResult = ReadLengthDelimitedStringPool(Header_.LabelNamesSize); + ui64 labelSizeTotal = (ui64(Header_.LabelNamesSize) + Header_.LabelValuesSize) + * (sizeof(std::pair) + sizeof(TStringBuf)); + DECODE_ENSURE(labelSizeTotal <= LABEL_SIZE_LIMIT, "label pool limit exceeded by segment metadata"); + + auto namesResult = ReadLengthDelimitedStringPool(Header_.LabelNamesSize, labelSizeTotal); namesBuf = std::move(namesResult.first); - auto valuesResult = ReadLengthDelimitedStringPool(Header_.LabelValuesSize); + auto valuesResult = ReadLengthDelimitedStringPool(Header_.LabelValuesSize, labelSizeTotal); valuesBuf = std::move(valuesResult.first); TStringPool labelNames(namesBuf.data(), namesResult.second); @@ -306,13 +310,15 @@ namespace NMonitoring { return ReadVarUInt32(In_); } - std::pair, TVector>> ReadLengthDelimitedStringPool(ui32 count) { + std::pair, TVector>> ReadLengthDelimitedStringPool(ui32 count, ui64& labelSizeTotal) { TVector storage; TVector> segments; segments.reserve(count); for (ui32 i = 0; i < count; i++) { ui32 len = ReadVarint(); + DECODE_ENSURE(len <= LABEL_SIZE_LIMIT - labelSizeTotal, "label pool limit exceeded by string #" << i); + labelSizeTotal += len; ui32 offset = static_cast(storage.size()); storage.resize(offset + len); if (len > 0) { diff --git a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp index 035e48dab80..af2aae29f05 100644 --- a/library/cpp/monlib/encode/spack/spack_v1_ut.cpp +++ b/library/cpp/monlib/encode/spack/spack_v1_ut.cpp @@ -1402,6 +1402,67 @@ Y_UNIT_TEST_SUITE(TSpackTest) { UNIT_ASSERT_VALUES_EQUAL(consumer.Metrics[0].StartTimeSeconds, startTimeSeconds); } + Y_UNIT_TEST(V13AndV14RejectOversizedLabelCount) { + // EzoEARgAAAD///8tAQAAAFsAAAAIABGrAPY= + ui8 corpus[] = { + 0x13, 0x3a, 0x04, 0x01, 0x18, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x2d, 0x01, 0x00, 0x00, 0x00, + 0x5b, 0x00, 0x00, 0x00, 0x08, 0x00, 0x11, 0xab, + 0x00, 0xf6, + }; + for (ui8 version : {ui8{3}, ui8{4}}) { + corpus[2] = version; + TBuffer buffer; + buffer.Append(reinterpret_cast(corpus), sizeof(corpus)); + TBufferInput in(buffer); + TCollectingConsumer consumer; + UNIT_ASSERT_EXCEPTION_CONTAINS(DecodeSpackV1(&in, &consumer), TSpackDecodeError, "limit"); + } + } + + Y_UNIT_TEST(V13AndV14RejectCombinedLabelCount) { + for (auto version : {SV1_03, SV1_04}) { + TSpackHeader header; + header.Version = version; + header.TimePrecision = EncodeTimePrecision(ETimePrecision::SECONDS); + header.Compression = EncodeCompression(ECompression::IDENTITY); + header.LabelNamesSize = 3'000'000; + header.LabelValuesSize = 3'000'000; + header.MetricCount = 1; + header.PointsCount = 0; + + TBuffer buffer; + TBufferOutput out(buffer); + out.Write(&header, sizeof(header)); + TBufferInput in(buffer); + TCollectingConsumer consumer; + UNIT_ASSERT_EXCEPTION_CONTAINS(DecodeSpackV1(&in, &consumer), TSpackDecodeError, "limit"); + } + } + + Y_UNIT_TEST(V13AndV14RejectOversizedLabelString) { + for (auto version : {SV1_03, SV1_04}) { + TSpackHeader header; + header.Version = version; + header.TimePrecision = EncodeTimePrecision(ETimePrecision::SECONDS); + header.Compression = EncodeCompression(ECompression::IDENTITY); + header.LabelNamesSize = 1; + header.LabelValuesSize = 0; + header.MetricCount = 1; + header.PointsCount = 0; + + TBuffer buffer; + TBufferOutput out(buffer); + out.Write(&header, sizeof(header)); + const ui8 oversizedLength[] = {0x80, 0x80, 0x80, 0x80, 0x01}; + out.Write(oversizedLength, sizeof(oversizedLength)); + + TBufferInput in(buffer); + TCollectingConsumer consumer; + UNIT_ASSERT_EXCEPTION_CONTAINS(DecodeSpackV1(&in, &consumer), TSpackDecodeError, "limit"); + } + } + Y_UNIT_TEST(V12MissingNameForOneMetric) { TBuffer b; TBufferOutput out(b); diff --git a/library/cpp/resource/registry.cpp b/library/cpp/resource/registry.cpp index 00badd0fe31..2538d24b133 100644 --- a/library/cpp/resource/registry.cpp +++ b/library/cpp/resource/registry.cpp @@ -1,6 +1,7 @@ #include "registry.h" #include "resource.h" +#include "static_hash.h" #include #include @@ -9,11 +10,16 @@ #include #include #include +#include #include using namespace NResource; using namespace NBlockCodecs; +// Static resource table assembled at link time by library/python/runtime_py3/pyres_ld_plugin.py. +// Binaries without packed python resources get the empty one from static_blob_default.cpp. +extern "C" const char __pyres_static_blob[]; + namespace { inline const ICodec* GetCodec() noexcept { static const ICodec* ret = Codec("zstd08_5"); @@ -21,6 +27,14 @@ namespace { return ret; } + inline const NStaticHash::TTable* StaticTable() noexcept { + return NStaticHash::TTable::FromBlob(__pyres_static_blob); + } + + inline TStringBuf AsBuf(std::string_view s) noexcept { + return TStringBuf(s.data(), s.size()); + } + typedef std::pair TDescriptor; [[noreturn]] @@ -50,14 +64,39 @@ namespace { } } + TString DecompressDiag(const TStringBuf key, const TStringBuf data) { + // temporary + // https://st.yandex-team.ru/DEVTOOLS-3985 + try { + return Decompress(data); + } catch (const yexception& e) { + if (GetEnv("RESOURCE_DECOMPRESS_DIAG")) { + Cerr << "Can't decompress resource " << key << Endl << e.what() << Endl; + } + throw e; + } + } + struct TStore final: public IStore, public absl::flat_hash_map { static inline ui64 ToK(TStringBuf k) { return NHashPrivate::ComputeStringHash(k.data(), k.size()); } + bool FindStatic(ui64 kk, TStringBuf key, NStaticHash::TRecord* out) const noexcept { + return S_ && S_->FindByHash(kk, std::string_view(key.data(), key.size()), out); + } + void Store(const TStringBuf key, const TStringBuf data) override { auto kk = ToK(key); + if (NStaticHash::TRecord rec; FindStatic(kk, key, &rec)) { + if (AsBuf(rec.Value) != data) { + ReportRedefinitionError(key, data, kk, TDescriptor(AsBuf(rec.Key), AsBuf(rec.Value))); + Y_UNREACHABLE(); + } + return; // already served by the static table + } + const auto [it, unique] = try_emplace(kk, nullptr); if (!unique) { const auto& [_, value] = *it->second; @@ -70,26 +109,23 @@ namespace { it->second = &D_.back(); } - Y_ABORT_UNLESS(size() == Count(), "size mismatch"); + Y_ABORT_UNLESS(size() == D_.size(), "size mismatch"); } bool Has(const TStringBuf key) const override { - return contains(ToK(key)); + const auto kk = ToK(key); + NStaticHash::TRecord rec; + return contains(kk) || FindStatic(kk, key, &rec); } bool FindExact(const TStringBuf key, TString* out) const override { - if (auto res = find(ToK(key)); res != end()) { - // temporary - // https://st.yandex-team.ru/DEVTOOLS-3985 - try { - *out = Decompress(res->second->second); - } catch (const yexception& e) { - if (GetEnv("RESOURCE_DECOMPRESS_DIAG")) { - Cerr << "Can't decompress resource " << key << Endl << e.what() << Endl; - } - throw e; - } - + const auto kk = ToK(key); + if (auto res = find(kk); res != end()) { + *out = DecompressDiag(key, res->second->second); + return true; + } + if (NStaticHash::TRecord rec; FindStatic(kk, key, &rec)) { + *out = DecompressDiag(key, AsBuf(rec.Value)); return true; } @@ -99,32 +135,39 @@ namespace { void FindMatch(const TStringBuf subkey, IMatch& cb) const override { for (const auto& it : D_) { if (it.first.StartsWith(subkey)) { - // temporary - // https://st.yandex-team.ru/DEVTOOLS-3985 - try { - const TResource res = { - it.first, Decompress(it.second)}; + const TResource res = {it.first, DecompressDiag(it.first, it.second)}; + cb.OnMatch(res); + } + } + if (S_) { + // TODO: records are sorted by key, the prefix range can be found by binary search + for (size_t i = 0, n = S_->Count(); i < n; ++i) { + const auto rec = S_->ByIndex(i); + const TStringBuf key = AsBuf(rec.Key); + if (key.StartsWith(subkey)) { + const TResource res = {key, DecompressDiag(key, AsBuf(rec.Value))}; cb.OnMatch(res); - } catch (const yexception& e) { - if (GetEnv("RESOURCE_DECOMPRESS_DIAG")) { - Cerr << "Can't decompress resource " << it.first << Endl << e.what() << Endl; - } - throw e; } } } } size_t Count() const noexcept override { - return D_.size(); + return D_.size() + (S_ ? S_->Count() : 0); } TStringBuf KeyByIndex(size_t idx) const override { - return D_.at(idx).first; + if (idx < D_.size()) { + return D_.at(idx).first; + } + idx -= D_.size(); + Y_ENSURE(S_ && idx < S_->Count(), "resource index out of range"); + return AsBuf(S_->ByIndex(idx).Key); } typedef TDeque TDescriptors; TDescriptors D_; + const NStaticHash::TTable* S_ = StaticTable(); }; } diff --git a/library/cpp/resource/static_blob_default.cpp b/library/cpp/resource/static_blob_default.cpp new file mode 100644 index 00000000000..1f1fa24587d --- /dev/null +++ b/library/cpp/resource/static_blob_default.cpp @@ -0,0 +1,5 @@ +// Fallback definition of the static resource table referenced from registry.cpp: an empty table +// (no magic, so it is ignored). library/python/runtime_py3/pyres_ld_plugin.py puts an object with +// the real table in front of all the libraries on the link command line; then the linker never +// pulls this object out of the archive. It must stay alone in its translation unit for that. +extern "C" alignas(16) const char __pyres_static_blob[16] = {}; diff --git a/library/cpp/resource/static_hash.h b/library/cpp/resource/static_hash.h new file mode 100644 index 00000000000..c1fc362fb71 --- /dev/null +++ b/library/cpp/resource/static_hash.h @@ -0,0 +1,122 @@ +#pragma once + +// Read-only view of the static resource table produced at link time by +// library/python/runtime_py3/pyres_ld_plugin.py (see the layout description there). +// Dependency-free on purpose: usable from the registry and from standalone tests. +// +// header (64 bytes) +// 0 char[8] magic "YPYRES01" +// 8 u64 count +// 16 u64 nbuckets power of two, >= 2 * count +// 24 u64 buckets_off nbuckets x (u64 hash, u64 key_off); key_off == 0 -> empty +// 32 u64 order_off count x u64 key_off, sorted by key bytes +// 40 u64 keys_off key records (8-aligned): u32 klen, u32 vlen, u64 val_off, key bytes +// 48 u64 total_size +// 56 u64 values_off raw values (compressed as in NResource::Compress) at val_off +// +// Keys are kept apart from values so that enumerating keys does not page in the values. +// +// Hash is CityHash64 of the key, i.e. NHashPrivate::ComputeStringHash on 64-bit platforms. + +#include +#include +#include +#include + +namespace NResource::NStaticHash { + constexpr std::string_view MAGIC = "YPYRES01"; + + struct TRecord { + std::string_view Key; + std::string_view Value; + }; + + class TTable { + public: + static const TTable* FromBlob(const void* blob) noexcept { + if (!blob) { + return nullptr; + } + const auto* table = static_cast(blob); + if (memcmp(table->Magic_, MAGIC.data(), MAGIC.size()) != 0) { + return nullptr; + } + return table; + } + + size_t Count() const noexcept { + return Count_; + } + + // Records in key order. + TRecord ByIndex(size_t idx) const noexcept { + return RecordAt(Load64(Base() + OrderOff_ + 8 * idx)); + } + + bool FindByHash(uint64_t hash, std::string_view key, TRecord* out) const noexcept { + if (Count_ == 0) { + return false; + } + const uint64_t mask = NBuckets_ - 1; + const char* buckets = Base() + BucketsOff_; + for (uint64_t i = hash & mask;; i = (i + 1) & mask) { + const char* bucket = buckets + 16 * i; + const uint64_t off = Load64(bucket + 8); + if (off == 0) { + return false; + } + if (Load64(bucket) == hash) { + const TRecord rec = RecordAt(off); + if (rec.Key == key) { + *out = rec; + return true; + } + } + } + } + + size_t SizeInBytes() const noexcept { + return TotalSize_; + } + + private: + const char* Base() const noexcept { + return reinterpret_cast(this); + } + + static uint64_t Load64(const char* p) noexcept { + uint64_t v; + memcpy(&v, p, sizeof(v)); + return v; + } + + static uint32_t Load32(const char* p) noexcept { + uint32_t v; + memcpy(&v, p, sizeof(v)); + return v; + } + + TRecord RecordAt(uint64_t off) const noexcept { + const char* p = Base() + off; + const uint32_t klen = Load32(p); + const uint32_t vlen = Load32(p + 4); + const uint64_t voff = Load64(p + 8); + return {std::string_view(p + 16, klen), std::string_view(Base() + voff, vlen)}; + } + + TTable() = delete; + TTable(const TTable&) = delete; + ~TTable() = delete; + + char Magic_[8]; + uint64_t Count_; + uint64_t NBuckets_; + uint64_t BucketsOff_; + uint64_t OrderOff_; + [[maybe_unused]] uint64_t KeysOff_; + uint64_t TotalSize_; + [[maybe_unused]] uint64_t ValuesOff_; + }; + + static_assert(sizeof(TTable) == 64); +} // namespace NResource::NStaticHash From a3b781b56bfa4a9a97e2858b92da58505d0d21eb Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Wed, 11 Nov 2026 17:07:32 +0000 Subject: [PATCH 06/17] Support sequence replication (#58220) --- .github/last_commit.txt | 2 +- include/ydb-cpp-sdk/client/table/table.h | 10 ++++ src/client/table/table.cpp | 69 +++++++++++++++++++----- tests/unit/client/table/table_ut.cpp | 53 +++++++++++++++++- 4 files changed, 119 insertions(+), 15 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 98b7d36f7d9..c0ef85c71f2 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -a5253b0ae0647e2650b30b7042bec226628a187f +b46b9468cadb65229107b678556bd649cd7ea674 diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 4ab52439c92..3688ffb8181 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -128,6 +128,16 @@ struct TSequenceDescription { bool NextUsed; }; std::optional SetVal; + // Name of the sequence: serial columns use `_serial_column_` by default, while + // sequences provisioned for index columns have their own names. + std::string Name; + std::optional MinValue; + std::optional MaxValue; + std::optional StartValue; + std::optional Cache; + std::optional Increment; + std::optional Cycle; + std::optional DataType; }; struct TTableColumn { diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 568f8f4e7b1..63216dbf85d 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -381,16 +381,37 @@ class TTableDescription::TImpl { std::optional sequenceDescription; switch (col.default_value_case()) { case Ydb::Table::ColumnMeta::kFromSequence: { - if (col.from_sequence().name() == "_serial_column_" + col.name()) { - TSequenceDescription currentSequenceDescription; - if (col.from_sequence().has_set_val()) { - TSequenceDescription::TSetVal setVal; - setVal.NextUsed = col.from_sequence().set_val().next_used(); - setVal.NextValue = col.from_sequence().set_val().next_value(); - currentSequenceDescription.SetVal = std::move(setVal); - } - sequenceDescription = std::move(currentSequenceDescription); + const auto& fromSequence = col.from_sequence(); + TSequenceDescription currentSequenceDescription; + currentSequenceDescription.Name = fromSequence.name(); + if (fromSequence.has_set_val()) { + TSequenceDescription::TSetVal setVal; + setVal.NextUsed = fromSequence.set_val().next_used(); + setVal.NextValue = fromSequence.set_val().next_value(); + currentSequenceDescription.SetVal = std::move(setVal); } + if (fromSequence.has_min_value()) { + currentSequenceDescription.MinValue = fromSequence.min_value(); + } + if (fromSequence.has_max_value()) { + currentSequenceDescription.MaxValue = fromSequence.max_value(); + } + if (fromSequence.has_start_value()) { + currentSequenceDescription.StartValue = fromSequence.start_value(); + } + if (fromSequence.has_cache()) { + currentSequenceDescription.Cache = fromSequence.cache(); + } + if (fromSequence.has_increment()) { + currentSequenceDescription.Increment = fromSequence.increment(); + } + if (fromSequence.has_cycle()) { + currentSequenceDescription.Cycle = fromSequence.cycle(); + } + if (fromSequence.has_data_type()) { + currentSequenceDescription.DataType = TType(fromSequence.data_type()); + } + sequenceDescription = std::move(currentSequenceDescription); break; } default: break; @@ -1073,13 +1094,35 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con protoColumn.set_not_null(column.NotNull.value()); } if (column.SequenceDescription.has_value()) { + const auto& sequence = *column.SequenceDescription; auto* fromSequence = protoColumn.mutable_from_sequence(); - if (column.SequenceDescription->SetVal.has_value()) { + fromSequence->set_name(sequence.Name.empty() ? "_serial_column_" + column.Name : sequence.Name); + if (sequence.MinValue) { + fromSequence->set_min_value(*sequence.MinValue); + } + if (sequence.MaxValue) { + fromSequence->set_max_value(*sequence.MaxValue); + } + if (sequence.StartValue) { + fromSequence->set_start_value(*sequence.StartValue); + } + if (sequence.Cache) { + fromSequence->set_cache(*sequence.Cache); + } + if (sequence.Increment) { + fromSequence->set_increment(*sequence.Increment); + } + if (sequence.Cycle) { + fromSequence->set_cycle(*sequence.Cycle); + } + if (sequence.DataType) { + fromSequence->mutable_data_type()->CopyFrom(sequence.DataType->GetProto()); + } + if (sequence.SetVal.has_value()) { auto* setVal = fromSequence->mutable_set_val(); - setVal->set_next_value(column.SequenceDescription->SetVal->NextValue); - setVal->set_next_used(column.SequenceDescription->SetVal->NextUsed); + setVal->set_next_value(sequence.SetVal->NextValue); + setVal->set_next_used(sequence.SetVal->NextUsed); } - fromSequence->set_name("_serial_column_" + column.Name); } } diff --git a/tests/unit/client/table/table_ut.cpp b/tests/unit/client/table/table_ut.cpp index 719027032af..de16258e223 100644 --- a/tests/unit/client/table/table_ut.cpp +++ b/tests/unit/client/table/table_ut.cpp @@ -1063,7 +1063,6 @@ TEST(VectorIndexSettings, ReducedPrecisionRoundTrip) { } } - TEST(VectorIndexSettings, Float8RoundTrip) { using TSettings = NYdb::NTable::TVectorIndexSettings; for (const auto& [type, protoType, name] : { @@ -1362,3 +1361,55 @@ TEST(VectorIndexSettings, HnswDiagnosticOutputIncludesSettings) { const NTable::TIndexDescription unconfigured("vidx", NTable::EIndexType::GlobalHnsw, {"emb"}); EXPECT_EQ(unconfigured.ToString().find("vector_settings:"), std::string::npos); } + +TEST(TTableDescription, SequenceDescriptionRoundTrip) { + using namespace NYdb::NTable; + + Ydb::Table::DescribeTableResult desc; + + auto* column = desc.add_columns(); + column->set_name("__ydb_row_id"); + column->mutable_type()->set_type_id(Ydb::Type::UINT64); + column->set_not_null(true); + + auto* source = column->mutable_from_sequence(); + source->set_name("_seq___ydb_row_id"); + source->set_min_value(1); + source->set_max_value(std::numeric_limits::max()); + source->set_start_value(1); + source->set_cache(100); + source->set_increment(5); + source->set_cycle(true); + source->mutable_data_type()->set_type_id(Ydb::Type::UINT64); + source->mutable_set_val()->set_next_value(42); + source->mutable_set_val()->set_next_used(true); + + desc.add_primary_key("__ydb_row_id"); + + TTableDescription table(std::move(desc), TDescribeTableSettings()); + + Ydb::Table::CreateTableRequest create; + table.SerializeTo(create); + + ASSERT_EQ(create.columns_size(), 1); + const auto& result = create.columns(0).from_sequence(); + + EXPECT_EQ(result.name(), "_seq___ydb_row_id"); + ASSERT_TRUE(result.has_min_value()); + EXPECT_EQ(result.min_value(), 1); + ASSERT_TRUE(result.has_max_value()); + EXPECT_EQ(result.max_value(), std::numeric_limits::max()); + ASSERT_TRUE(result.has_start_value()); + EXPECT_EQ(result.start_value(), 1); + ASSERT_TRUE(result.has_cache()); + EXPECT_EQ(result.cache(), 100u); + ASSERT_TRUE(result.has_increment()); + EXPECT_EQ(result.increment(), 5); + ASSERT_TRUE(result.has_cycle()); + EXPECT_TRUE(result.cycle()); + ASSERT_TRUE(result.has_data_type()); + EXPECT_EQ(result.data_type().type_id(), Ydb::Type::UINT64); + ASSERT_TRUE(result.has_set_val()); + EXPECT_EQ(result.set_val().next_value(), 42); + EXPECT_TRUE(result.set_val().next_used()); +} From 02fb97735de3bf77ec2215b4b94d8fde02bfd019 Mon Sep 17 00:00:00 2001 From: Alexander Rutkovsky Date: Wed, 11 Nov 2026 17:07:47 +0000 Subject: [PATCH 07/17] Bridge: roll a converted or prepared cluster back to ordinary mode (#58747) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 ++ src/api/grpc/draft/ydb_bridge_v1.proto | 6 ++++ src/api/protos/draft/ydb_bridge.proto | 50 ++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index c0ef85c71f2..f63ff9106c9 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -b46b9468cadb65229107b678556bd649cd7ea674 +b0e1eb15a9962dbba38489a9d7bf46a5fc5acde4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a8cf7a098f..e0a08226f23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Added draft `TBridgeClient::Rollback()` to roll a cluster converted into a bridged one back to ordinary mode, and `TBridgeClient::GetRollbackStatus()` to show the progress of the rollback. + * Topic and PersQueue writers now schedule RAW and pre-encoded messages on the configured compression executor, reducing request construction on the caller thread. * Added optional `TReplicationDescription::GetTransactions()` to report assigned global-consistency replication transactions and versions pending transaction-ID assignment. diff --git a/src/api/grpc/draft/ydb_bridge_v1.proto b/src/api/grpc/draft/ydb_bridge_v1.proto index ecf7d4a6284..bf685db85a3 100644 --- a/src/api/grpc/draft/ydb_bridge_v1.proto +++ b/src/api/grpc/draft/ydb_bridge_v1.proto @@ -31,4 +31,10 @@ service BridgeService { // Remove a DISCONNECTED pile from the cluster along with all its data, so that it becomes INACTIVE rpc RemovePile(Ydb.Bridge.RemovePileRequest) returns (Ydb.Bridge.RemovePileResponse); + // Report how far the rollback of a cluster prepared for bridge mode back to ordinary mode has gone + rpc GetRollbackStatus(Ydb.Bridge.GetRollbackStatusRequest) returns (Ydb.Bridge.GetRollbackStatusResponse); + + // Roll a cluster converted from an ordinary one back to it, while it has the only pile, the original one + rpc Rollback(Ydb.Bridge.RollbackRequest) returns (Ydb.Bridge.RollbackResponse); + } diff --git a/src/api/protos/draft/ydb_bridge.proto b/src/api/protos/draft/ydb_bridge.proto index 490af48baec..9b397dc053f 100644 --- a/src/api/protos/draft/ydb_bridge.proto +++ b/src/api/protos/draft/ydb_bridge.proto @@ -93,6 +93,8 @@ message PendingPileOperation { ADD = 1; // The pile is being removed from the cluster: its storage groups are being deleted REMOVE = 2; + // The cluster is being rolled back to ordinary one: its storage groups are being made ordinary ones + ROLLBACK = 3; } string pile_name = 1; Kind kind = 2; @@ -177,6 +179,54 @@ message RemovePileResponse { message RemovePileResult { } +// Report how far the rollback of a cluster prepared for bridge mode has gone. Such cluster (its nodes run with a +// single-pile bridge config, but it has not been converted into a bridged one) gets back to ordinary mode when the bridge +// config, along with the pile names of hosts, is removed from the cluster configuration in a single change, and every +// node is restarted without them (dynamic nodes without their pile names too); Hive and NodeBroker drop the piles of +// nodes by themselves then. Dynamic nodes not connected to the cluster are not reported. Administrator privileges are +// required. +message GetRollbackStatusRequest { + Ydb.Operations.OperationParams operation_params = 1; +} + +message GetRollbackStatusResponse { + Ydb.Operations.Operation operation = 1; +} + +message GetRollbackStatusResult { + // The cluster is a bridged one; see Rollback, the cluster state goes away when it finishes. + bool bridged = 1; + // Connected nodes still running with a bridge config; they have to be restarted without one. + repeated uint32 bridge_config_node_ids = 2; + // Static nodes that have not reported their config, as they are not running or not connected. + repeated uint32 unreported_static_node_ids = 3; + // The cluster is not bridged, and every node has reported running without a bridge config. + bool complete = 4; + // The rollback of the bridged cluster is in progress: storage groups are being made ordinary ones. + bool rollback_in_progress = 5; + // Why the last attempt to make storage groups ordinary ones has failed; it is retried meanwhile. + string error_reason = 6; +} + +// Roll a cluster converted from an ordinary one back to it, while it has the only pile, the original one, and every +// storage group is inline in that pile (has its own VDisks there and no parts elsewhere). Static storage groups become +// ordinary ones by the call, the storage controller does the same to its groups afterwards, then the cluster state is +// removed; see GetRollbackStatusResult. The cluster is prepared for bridge mode then, and gets back to ordinary mode +// as GetRollbackStatusRequest describes. +message RollbackRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Don't ask the storage controller whether it can make its groups ordinary ones before static groups are made such. + // This is for the case when it can't start, as it relies on static groups; it reports what blocks it then. + bool skip_bsc_validation = 2; +} + +message RollbackResponse { + Ydb.Operations.Operation operation = 1; +} + +message RollbackResult { +} + message BridgeHistoryEvent { // Timestamp of the event google.protobuf.Timestamp timestamp = 1; From 93ccc92849c06497fbdcbf934809fa6b74974fb4 Mon Sep 17 00:00:00 2001 From: mregrock Date: Wed, 11 Nov 2026 17:07:54 +0000 Subject: [PATCH 08/17] Expose bridge synchronization diagnostics through RPC and CLI (#58794) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + src/api/grpc/draft/ydb_bridge_v1.proto | 9 ++ src/api/protos/draft/ydb_bridge.proto | 177 ++++++++++++++++++++++++- 4 files changed, 184 insertions(+), 6 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index f63ff9106c9..45f4d2123ce 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -b0e1eb15a9962dbba38489a9d7bf46a5fc5acde4 +0fc7e20b3d5f7d10be6b27828f098e8a154f1150 diff --git a/CHANGELOG.md b/CHANGELOG.md index e0a08226f23..8ca18ac571b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,10 @@ * Added optional `TReplicationDescription::GetTransactions()` to report assigned global-consistency replication transactions and versions pending transaction-ID assignment. * Added `TGetClusterStateSettings::IncludePileConnectivity()` and `TGetClusterStateResult::GetPileConnectivity()` for optional bridge static-node connectivity reports. +* Added `TGetClusterStateSettings::IncludeUnbridgedStatus()` and `TGetClusterStateResult::GetUnbridgedStatus()` to report node configuration when the cluster has no bridge state. * Added `TUpdateClusterStateSettings::ExpectedGeneration()` to apply a bridge cluster state update only if the cluster state still has the given generation. +* Added draft bridge synchronization APIs for pile, database, and group diagnostics. * Added draft `TBridgeClient::SetDatabaseBridgeMode()` to atomically change a database's bridge mode. `TSetDatabaseBridgeModeSettings::RetainPileName()` selects the pile to retain when disabling bridge mode. diff --git a/src/api/grpc/draft/ydb_bridge_v1.proto b/src/api/grpc/draft/ydb_bridge_v1.proto index bf685db85a3..ab02cf4515f 100644 --- a/src/api/grpc/draft/ydb_bridge_v1.proto +++ b/src/api/grpc/draft/ydb_bridge_v1.proto @@ -18,6 +18,15 @@ service BridgeService { // Useful for monitoring and decision-making before state changes rpc GetClusterState(Ydb.Bridge.GetClusterStateRequest) returns (Ydb.Bridge.GetClusterStateResponse); + // List per-pile database synchronization counters. + rpc ListPileSynchronization(Ydb.Bridge.ListPileSynchronizationRequest) returns (Ydb.Bridge.ListPileSynchronizationResponse); + + // List synchronization summaries by storage-owning database and pile. + rpc ListDatabaseSynchronization(Ydb.Bridge.ListDatabaseSynchronizationRequest) returns (Ydb.Bridge.ListDatabaseSynchronizationResponse); + + // List all group synchronization, optionally restricted to a database or storage pool. + rpc ListGroupSynchronization(Ydb.Bridge.ListGroupSynchronizationRequest) returns (Ydb.Bridge.ListGroupSynchronizationResponse); + // Update cluster state by providing a list of desired pile states // This is the main operation for failover and promotion scenarios rpc UpdateClusterState(Ydb.Bridge.UpdateClusterStateRequest) returns (Ydb.Bridge.UpdateClusterStateResponse); diff --git a/src/api/protos/draft/ydb_bridge.proto b/src/api/protos/draft/ydb_bridge.proto index 9b397dc053f..add15a4c3c9 100644 --- a/src/api/protos/draft/ydb_bridge.proto +++ b/src/api/protos/draft/ydb_bridge.proto @@ -32,9 +32,12 @@ message SetDatabaseBridgeModeResult {} message GetClusterStateRequest { Ydb.Operations.OperationParams operation_params = 1; - // Also query static-node connectivity, without requiring BSC or tablets. + // For a bridge state snapshot, also query static-node connectivity through distconf. // An older node in the forwarding path may reject this optional query. bool include_pile_connectivity = 2; + + // If the cluster has no bridge state, collect the node configuration report. This requires administrator privileges. + bool include_unbridged_status = 3; } message GetClusterStateResponse { @@ -49,11 +52,175 @@ message GetClusterStateResult { // Set after the cluster has been converted from a non-bridged one, until all its storage groups are converted // too; no piles can be added to the cluster meanwhile bool conversion_in_progress = 3; - // Operation changing the set of piles of the cluster that is not finished yet + // Pending pile operation or rollback to ordinary mode. optional PendingPileOperation pending_pile_operation = 4; optional PileConnectivityReport pile_connectivity = 5; + // A report-only response for a cluster without bridge state. Pile states and generation apply to bridge snapshots. + UnbridgedStatus unbridged_status = 6; +} + +message UnbridgedStatus { + // Nodes that reported running with bridge config. + repeated uint32 bridge_config_node_ids = 1; + // Configured static nodes whose configuration was not observed. + repeated uint32 unreported_static_node_ids = 2; } +// Administrator-only diagnostics; complete results or an error. +message ListPileSynchronizationRequest { + Ydb.Operations.OperationParams operation_params = 1; +} + +message ListPileSynchronizationResponse { + Ydb.Operations.Operation operation = 1; +} + +message ListPileSynchronizationResult { + repeated PileSynchronization piles = 1; + uint64 cluster_state_generation = 2; + google.protobuf.Timestamp observed_at = 3; + // Pools whose database cannot be identified; excluded from totals. + uint64 unattributed_storage_pools = 4; +} + +message ListDatabaseSynchronizationRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Absolute path; serverless selects shared storage. + optional string database_path = 2; + // Unset: all piles; empty: the pile named "". + optional string pile_name = 3; + // Skip synchronized databases and those with no required groups. + bool only_incomplete = 4; + // Filter by the aggregate mode of all database pools. + optional StorageMode storage_mode = 5; +} + +message ListDatabaseSynchronizationResponse { + Ydb.Operations.Operation operation = 1; +} + +message ListDatabaseSynchronizationResult { + // Storage-owning databases, not all logical serverless databases. + repeated DatabaseSynchronization databases = 1; + uint64 cluster_state_generation = 2; + google.protobuf.Timestamp observed_at = 3; + // Resolved owner, also returned for an empty selection. + optional string storage_owner_database_path = 4; +} + +message ListGroupSynchronizationRequest { + Ydb.Operations.OperationParams operation_params = 1; + // Unset: all groups. Paths/names must be nonempty; ambiguous pool names require IDs. + oneof scope { + string database_path = 2; + string storage_pool_name = 3; + StoragePoolId storage_pool_id = 4; + } + // Unset: all piles; empty: the pile named "". + optional string pile_name = 5; + // Empty: all statuses; otherwise OR. Unspecified is invalid. + repeated SynchronizationStatus statuses = 6; +} + +message ListGroupSynchronizationResponse { + Ydb.Operations.Operation operation = 1; +} + +message ListGroupSynchronizationResult { + repeated GroupSynchronization groups = 1; + uint64 cluster_state_generation = 2; + google.protobuf.Timestamp observed_at = 3; + optional string storage_owner_database_path = 4; +} + +message StoragePoolId { + uint64 box_id = 1; + // Nonzero; box_id may be zero. + uint64 pool_id = 2; +} + +message StoragePool { + StoragePoolId id = 1; + string name = 2; +} + +enum StorageMode { + STORAGE_MODE_UNSPECIFIED = 0; + BRIDGED = 1; + ORDINARY = 2; + MIXED = 3; +} + +enum SynchronizationStatus { + SYNCHRONIZATION_STATUS_UNSPECIFIED = 0; + SYNCED = 1; + SYNCING = 2; + PENDING = 3; // Required, but not running. + // No bridge synchronization required, including ordinary storage and inactive piles. + NOT_REQUIRED = 4; + UNKNOWN = 5; // Insufficient observations. +} + +// Required groups only: required = synced + syncing + pending + unknown. +message GroupSyncCounts { + uint64 required = 1; + uint64 synced = 2; + uint64 syncing = 3; + uint64 pending = 4; + uint64 unknown = 5; +} + +// One known storage owner per pile; shared storage counts once. +// total = synced + syncing + pending + unknown + not_required. +message DatabaseSyncCounts { + uint64 total = 1; + uint64 synced = 2; + uint64 syncing = 3; + uint64 pending = 4; + uint64 unknown = 5; + uint64 not_required = 6; +} + +message PileSynchronization { + // Unset: unknown placement; empty: the real pile named "". + optional string pile_name = 1; + DatabaseSyncCounts databases = 2; +} + +// Last error; may remain while synchronization is running. +message SynchronizationError { + string message = 1; + google.protobuf.Timestamp occurred_at = 2; +} + +message DatabaseSynchronization { + // Storage owner path, if known. + optional string database_path = 1; + repeated StoragePool storage_pools = 2; + StorageMode storage_mode = 3; + // Unset if placement is unknown. + optional string pile_name = 4; + // Mixed mode counts only bridged pools. + GroupSyncCounts groups = 5; + SynchronizationError last_sync_error = 6; +} + +message GroupSynchronization { + // ID used by tablets. + uint32 group_id = 1; + // Physical group ID, if known. + optional uint32 physical_group_id = 2; + // Unset for groups outside dynamic pools. + StoragePool storage_pool = 3; + // One row per pile; unset if placement is unknown. + optional string pile_name = 4; + SynchronizationStatus sync_status = 5; + // Diagnostic phase; use sync_status for programmatic checks. + string sync_phase = 6; + SynchronizationError last_sync_error = 7; +} + + message PileConnectivity { string pile_name = 1; uint32 static_nodes = 2; @@ -200,7 +367,7 @@ message GetRollbackStatusResult { repeated uint32 bridge_config_node_ids = 2; // Static nodes that have not reported their config, as they are not running or not connected. repeated uint32 unreported_static_node_ids = 3; - // The cluster is not bridged, and every node has reported running without a bridge config. + // The cluster is ordinary, and all static and connected dynamic nodes run without bridge config. bool complete = 4; // The rollback of the bridged cluster is in progress: storage groups are being made ordinary ones. bool rollback_in_progress = 5; @@ -208,8 +375,8 @@ message GetRollbackStatusResult { string error_reason = 6; } -// Roll a cluster converted from an ordinary one back to it, while it has the only pile, the original one, and every -// storage group is inline in that pile (has its own VDisks there and no parts elsewhere). Static storage groups become +// Roll a converted cluster back to ordinary mode while the original pile is the only pile and every bridged storage +// group is inline in it (has its own VDisks there and no parts elsewhere). Static storage groups become // ordinary ones by the call, the storage controller does the same to its groups afterwards, then the cluster state is // removed; see GetRollbackStatusResult. The cluster is prepared for bridge mode then, and gets back to ordinary mode // as GetRollbackStatusRequest describes. From b768634ad2af15abcfc95019a3ba22a699a310ed Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 11 Nov 2026 17:08:01 +0000 Subject: [PATCH 09/17] [KIKIMR-27516] Fix C++ SDK Login retries before the first token (#58291) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + src/client/types/credentials/login/login.cpp | 4 +- tests/unit/client/login/login_ut.cpp | 155 +++++++++++++++++++ 4 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 tests/unit/client/login/login_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 45f4d2123ce..42fd4282367 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -0fc7e20b3d5f7d10be6b27828f098e8a154f1150 +72811be17fbce6e266846ac3bffbec71be735698 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ca18ac571b..1eb15e20e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Fixed static login credentials failing permanently when the initial Login operation returns a transient service error. + * Added draft `TBridgeClient::Rollback()` to roll a cluster converted into a bridged one back to ordinary mode, and `TBridgeClient::GetRollbackStatus()` to show the progress of the rollback. * Topic and PersQueue writers now schedule RAW and pre-encoded messages on the configured compression executor, reducing request construction on the caller thread. diff --git a/src/client/types/credentials/login/login.cpp b/src/client/types/credentials/login/login.cpp index 01afd3153e3..398cc92c7ab 100644 --- a/src/client/types/credentials/login/login.cpp +++ b/src/client/types/credentials/login/login.cpp @@ -75,7 +75,6 @@ class TLoginCredentialsProvider : public ICredentialsProvider, public std::enabl mutable std::mutex Mutex_; TInstant TokenRequestAt_; bool Requesting_ = false; - bool HasToken_ = false; bool Stopped_ = false; mutable NThreading::TPromise AuthInfo_; }; @@ -190,7 +189,7 @@ void TLoginCredentialsProvider::FinishRequest( { std::lock_guard lock(Mutex_); retry = (!status.Ok() && IsRetryable(status.Status)) || - (HasToken_ && status.Ok() && IsRetryableOperation(operationStatus)); + (status.Ok() && IsRetryableOperation(operationStatus)); if (retry) { Requesting_ = false; TokenRequestAt_ = TInstant::Now() + TDuration::Seconds(1); @@ -214,7 +213,6 @@ void TLoginCredentialsProvider::FinishRequest( { std::lock_guard lock(Mutex_); Requesting_ = false; - HasToken_ = true; TokenRequestAt_ = now + (ToInstant(GetTokenExpiresAt(token)) - now) / 2; promise = AuthInfo_; } diff --git a/tests/unit/client/login/login_ut.cpp b/tests/unit/client/login/login_ut.cpp new file mode 100644 index 00000000000..a90a2b6c69e --- /dev/null +++ b/tests/unit/client/login/login_ut.cpp @@ -0,0 +1,155 @@ +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +using namespace NYdb; + +namespace { + + class TDummyClient { + class TImpl: public TClientImplCommon { + public: + explicit TImpl(TDriver& driver) + : TClientImplCommon(CreateInternalInterface(driver), TCommonClientSettings()) + { + } + + std::shared_ptr GetCoreFacility() const { + return DbDriverState_; + } + }; + + public: + explicit TDummyClient(TDriver& driver) + : Impl_(std::make_shared(driver)) + { + } + + std::shared_ptr GetCoreFacility() const { + return Impl_->GetCoreFacility(); + } + + private: + std::shared_ptr Impl_; + }; + + class TLoginService final: public Ydb::Auth::V1::AuthService::Service { + public: + explicit TLoginService(Ydb::StatusIds::StatusCode firstStatus, + grpc::StatusCode firstTransportStatus = grpc::StatusCode::OK) + : FirstStatus(firstStatus) + , FirstTransportStatus(firstTransportStatus) + , Token(jwt::create() + .set_expires_at(std::chrono::system_clock::now() + std::chrono::hours(1)) + .sign(jwt::algorithm::none{})) + { + } + + grpc::Status Login(grpc::ServerContext*, const Ydb::Auth::LoginRequest* request, + Ydb::Auth::LoginResponse* response) override { + if (request->user() != "login-retry-user" || request->password() != "login-retry-password") { + CredentialsMatch = false; + } + const auto attempt = ++Requests; + if (attempt == 1 && !ReleaseFirst.WaitT(TDuration::Seconds(10))) { + return grpc::Status(grpc::StatusCode::DEADLINE_EXCEEDED, "Test response release timed out"); + } + if (attempt == 1 && FirstTransportStatus != grpc::StatusCode::OK) { + return grpc::Status(FirstTransportStatus, "Test transport failure"); + } + auto* operation = response->mutable_operation(); + operation->set_ready(true); + operation->set_status(attempt == 1 ? FirstStatus : Ydb::StatusIds::SUCCESS); + if (attempt > 1) { + Ydb::Auth::LoginResult result; + result.set_token(Token); + operation->mutable_result()->PackFrom(result); + } + return grpc::Status::OK; + } + + const Ydb::StatusIds::StatusCode FirstStatus; + const grpc::StatusCode FirstTransportStatus; + const std::string Token; + TManualEvent ReleaseFirst; + std::atomic_uint Requests{0}; + std::atomic_bool CredentialsMatch{true}; + }; + + template + void WithProvider(Ydb::StatusIds::StatusCode firstStatus, TCallback callback, + grpc::StatusCode firstTransportStatus = grpc::StatusCode::OK) { + NTesting::InitPortManagerFromEnv(); + const auto port = NTesting::GetFreePort(); + const std::string endpoint = "127.0.0.1:" + std::to_string(port); + TLoginService service(firstStatus, firstTransportStatus); + auto server = grpc::ServerBuilder() + .AddListeningPort(TStringType{endpoint}, grpc::InsecureServerCredentials()) + .RegisterService(&service) + .BuildAndStart(); + UNIT_ASSERT(server); + Y_DEFER { + service.ReleaseFirst.Signal(); + server->Shutdown(std::chrono::system_clock::now() + std::chrono::seconds(5)); + server->Wait(); + }; + + TDriver driver(TDriverConfig().SetEndpoint(endpoint).SetDatabase("/Root").SetDiscoveryMode(EDiscoveryMode::Off)); + TDummyClient client(driver); + auto provider = CreateLoginCredentialsProviderFactory({.User = "login-retry-user", .Password = "login-retry-password"})->CreateProvider(client.GetCoreFacility()); + Y_DEFER { + service.ReleaseFirst.Signal(); + provider.reset(); + driver.Stop(true); + }; + + // Capture this future before either response can complete it. Retrying by + // obtaining a new future/provider would hide a permanently failed first login. + const auto future = provider->GetAuthInfoAsync(); + service.ReleaseFirst.Signal(); + callback(service, future); + } + +} // namespace + +Y_UNIT_TEST_SUITE(Login) { + Y_UNIT_TEST(InitialTransientOperationRetries) { + WithProvider(Ydb::StatusIds::UNAVAILABLE, [](const auto& service, const auto& future) { + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(10)), "Initial login did not finish after a transient operation failure"); + UNIT_ASSERT_C(!future.HasException(), "Cold Login operation UNAVAILABLE permanently failed the original auth future"); + UNIT_ASSERT_VALUES_EQUAL(future.GetValueSync(), service.Token); + UNIT_ASSERT(service.Requests.load() >= 2); + UNIT_ASSERT(service.CredentialsMatch.load()); + }); + } + + Y_UNIT_TEST(InitialUnauthorizedFails) { + WithProvider(Ydb::StatusIds::UNAUTHORIZED, [](const auto& service, const auto& future) { + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(10)), "Rejected credentials left the initial auth future pending"); + UNIT_ASSERT(future.HasException()); + UNIT_ASSERT(service.Requests.load() >= 1); + UNIT_ASSERT(service.CredentialsMatch.load()); + }); + } + + Y_UNIT_TEST(InitialTransportUnauthenticatedFails) { + WithProvider(Ydb::StatusIds::SUCCESS, [](const auto& service, const auto& future) { + UNIT_ASSERT_C(future.Wait(TDuration::Seconds(10)), "Transport rejection left the initial auth future pending"); + UNIT_ASSERT(future.HasException()); + UNIT_ASSERT_VALUES_EQUAL(service.Requests.load(), 1); + UNIT_ASSERT(service.CredentialsMatch.load()); + }, grpc::StatusCode::UNAUTHENTICATED); + } +} // Y_UNIT_TEST_SUITE(Login) From f1229bda1a326678740e07bde227f71f6c0e9da0 Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:08:08 +0000 Subject: [PATCH 10/17] Preserve absolute Topic paths in the C++ SDK (#57964) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + src/client/topic/impl/topic_path.h | 10 +++++ src/client/topic/impl/write_session_impl.cpp | 6 +-- src/client/topic/ut/topic_path_ut.cpp | 40 ++++++++++++++++++++ 5 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 src/client/topic/impl/topic_path.h create mode 100644 src/client/topic/ut/topic_path_ut.cpp diff --git a/.github/last_commit.txt b/.github/last_commit.txt index 42fd4282367..f9bc968ba82 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -72811be17fbce6e266846ac3bffbec71be735698 +74f76f9ad62f41d568b3e4d17529bf59b6a35b99 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eb15e20e3c..ef35c61b7bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ * Added `TUpdateClusterStateSettings::ExpectedGeneration()` to apply a bridge cluster state update only if the cluster state still has the given generation. * Added draft bridge synchronization APIs for pile, database, and group diagnostics. +* Fixed Topic write-session path handling: paths starting with `/` remain absolute, and relative database names are left for server-side resolution. + * Added draft `TBridgeClient::SetDatabaseBridgeMode()` to atomically change a database's bridge mode. `TSetDatabaseBridgeModeSettings::RetainPileName()` selects the pile to retain when disabling bridge mode. * Fixed PersQueue topic settings losing replication-consumer and consumer-offset-mirroring attributes when copied from DescribeTopic into CreateTopic or AlterTopic settings. diff --git a/src/client/topic/impl/topic_path.h b/src/client/topic/impl/topic_path.h new file mode 100644 index 00000000000..ab3bae77797 --- /dev/null +++ b/src/client/topic/impl/topic_path.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +namespace NYdb::inline V3::NTopic { + + std::string FullTopicPath(const std::string& dbPath, std::string_view topic); + +} // namespace NYdb::inline V3::NTopic diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index 97349183652..eb9d55fbf88 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -1,6 +1,7 @@ #include "write_session_impl.h" #include "deferred_publication_ack_tracker.h" +#include "topic_path.h" #include #include @@ -332,7 +333,7 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat } std::string FullTopicPath(const std::string& dbPath, std::string_view topic) { - if (topic.starts_with(dbPath)) { + if (!dbPath.starts_with('/') || topic.starts_with('/')) { return std::string(topic); } std::string full; @@ -341,9 +342,6 @@ std::string FullTopicPath(const std::string& dbPath, std::string_view topic) { if (!full.ends_with('/')) { full.push_back('/'); } - if (topic.starts_with('/')) { - topic = topic.substr(1); - } full.append(topic); return full; } diff --git a/src/client/topic/ut/topic_path_ut.cpp b/src/client/topic/ut/topic_path_ut.cpp new file mode 100644 index 00000000000..976acdd7f65 --- /dev/null +++ b/src/client/topic/ut/topic_path_ut.cpp @@ -0,0 +1,40 @@ +#include + +#include + +#include + +namespace NYdb::inline V3::NTopic::NTests { + +Y_UNIT_TEST_SUITE(WriteSessionPath) { + Y_UNIT_TEST(AbsoluteDatabase) { + for (const auto& database : {"/Root/mydb", "/Root/mydb/", "//Root/mydb"}) { + for (const auto& path : {"Root/mydb/topic", "Root/other/topic", "Root/mydb2/topic", "Root"}) { + const auto separator = std::string(database).ends_with('/') ? "" : "/"; + UNIT_ASSERT_VALUES_EQUAL_C(FullTopicPath(database, path), std::string(database) + separator + path, path); + } + for (const auto& path : {"/Root/mydb/topic", "/Root/mydb2/topic", "/Other/topic", "//Root/mydb/topic"}) { + UNIT_ASSERT_VALUES_EQUAL_C(FullTopicPath(database, path), path, path); + } + } + } + + Y_UNIT_TEST(RelativeTopicIsJoinedToAbsoluteDatabase) { + for (const auto& database : {"/Root/mydb", "/Root/mydb/"}) { + for (const auto& path : {"topic", "mydb/topic", "Root2/topic", "root/topic"}) { + UNIT_ASSERT_VALUES_EQUAL_C(FullTopicPath(database, path), std::string("/Root/mydb/") + path, path); + } + } + UNIT_ASSERT_VALUES_EQUAL(FullTopicPath("/", "topic"), "/topic"); + } + + Y_UNIT_TEST(RelativeDatabaseLeavesTopicUnchanged) { + for (const auto& database : {"", "mydb", "Root/mydb", "Root/mydb/"}) { + for (const auto& path : {"", "topic", "mydb/topic", "Root/mydb/topic", "Root2/topic", "/Root/mydb/topic", "/Other/topic"}) { + UNIT_ASSERT_VALUES_EQUAL_C(FullTopicPath(database, path), path, database); + } + } + } +} + +} // namespace NYdb::inline V3::NTopic::NTests From 75f7812ae4f5b2cc59dc2c48b3f818d479d5f589 Mon Sep 17 00:00:00 2001 From: Sergey Date: Wed, 11 Nov 2026 17:08:16 +0000 Subject: [PATCH 11/17] Fix Topic read session panic during teardown on unrecoverable read error (#58908) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 + src/client/topic/impl/read_session_impl.ipp | 14 +++++ .../topic/ut/read_session_accounting_ut.cpp | 59 +++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index f9bc968ba82..f3f1f4f1132 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -74f76f9ad62f41d568b3e4d17529bf59b6a35b99 +0962f20baf13702db98024d8f1206a2f0398e670 diff --git a/CHANGELOG.md b/CHANGELOG.md index ef35c61b7bd..34bac278746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ * Supported Topic read sessions counting replication control messages as data offsets, which could block commits after an offset gap. +* Fixed a Topic read session crash (Panic in `OnUserRetrievedEvent`) when an unrecoverable stream read error tore down the session while data events were still queued for delivery; the teardown now releases the accounting for undelivered events exactly once. + ## v3.26.0 * Federated topic read sessions now follow federation membership. After the initial discovery, databases that join the federation are read and databases that leave it are dropped; partition sessions of a dropped database get `TPartitionSessionClosedEvent` with the `ConnectionLost` reason. A listed database that becomes unavailable keeps its subsession. The refresh period is `TFederatedTopicClientSettings::FederationDiscoveryInterval` (30 seconds by default). diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 040b92f11f1..b05d7a62303 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -2112,6 +2112,20 @@ void TSingleClusterReadSessionImpl::AbortSession(TASession TDeferredActions deferred; LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Closing session to cluster: " << closeEvent.DebugString()); + { + // The events queue below drains undelivered data events and defers + // OnUserRetrievedEvent for them. The session is being torn down, so + // the same data may already have been released by a concurrent path; + // mark the session aborting so the accounting guard in + // OnUserRetrievedEvent skips the second release. + // Only the flag is set here instead of calling AbortImpl: the direct + // read path calls this method while holding the direct reader lock, + // and AbortImpl would close the DirectReadSessionManager, which + // takes the same non-recursive lock. + std::lock_guard guard(Lock); + Aborting = true; + } + EventsQueue->Close(closeEvent, deferred); } diff --git a/src/client/topic/ut/read_session_accounting_ut.cpp b/src/client/topic/ut/read_session_accounting_ut.cpp index 1a49aa23c38..8dcfcaf73ed 100644 --- a/src/client/topic/ut/read_session_accounting_ut.cpp +++ b/src/client/topic/ut/read_session_accounting_ut.cpp @@ -60,6 +60,65 @@ namespace NYdb::inline V3::NTopic { queue.Cleanup(actions); } } + + // Regression test for YDBBUGS-1002. An unrecoverable read error takes + // the OnReadDone -> AbortSession(closeEvent) path, which drains the + // events queue and defers OnUserRetrievedEvent for undelivered ready + // data events. When the same bytes were concurrently released by a + // user-side retrieval, the session accounting must not underflow. + Y_UNIT_TEST(AbortSessionDrainingReadyEventsDoesNotPanicAfterConcurrentRelease) { + TReadSessionSettings settings; + auto counters = MakeIntrusive(); + MakeCountersNotNull(*counters); + settings.MaxMemoryUsageBytes(1_MB).Counters(counters); + + auto events = std::make_shared>(settings); + auto context = MakeWithCallbackContext>( + settings, "", "read-session", "", TLog{}, nullptr, events, nullptr, 1, 1); + auto session = context->TryGet(); + auto partition = MakeIntrusive>( + ui64{1}, "topic", "read-session", 0, 1, 0, std::nullopt, context); + + TPartitionData data; + auto* batch = data.add_batches(); + batch->set_codec(Ydb::Topic::CODEC_RAW); + batch->add_message_data()->set_data(std::string(10, 'a')); + batch->add_message_data()->set_data(std::string(20, 'b')); + auto info = std::make_shared>( + std::move(data), context, true); + + { + TDeferredActions actions; + UNIT_ASSERT(info->PlanDecompressionTasks(1.0, partition, actions)); + } + + { + TDeferredActions actions; + const auto estimatedSize = info->StartDecompressionTasks( + std::make_shared(), 1_MB, actions); + UNIT_ASSERT_VALUES_EQUAL(estimatedSize, 30); + // StartDecompressionTasksImpl normally reserves this estimated size. + session->OnDataDecompressed(0, 0, estimatedSize, 2); + } + + // Move the ready data events into the session events queue, + // as the session does when decompression completes. + events->SignalReadyEvents(partition); + + // The user concurrently retrieved and released the same 30 bytes + // (the GetEvents path), so the session has nothing left to release. + session->OnUserRetrievedEvent(30, 2); + + // Unrecoverable stream error on the gRPC thread: + // OnReadDone -> AbortSession(closeEvent). The queue cleanup must + // not release the same bytes a second time. + session->AbortSession(EStatus::ABORTED, "Test abort"); + + // The user still receives the session closed event. + auto closeEvent = events->GetEvent(); + UNIT_ASSERT(closeEvent.has_value()); + UNIT_ASSERT(std::holds_alternative(*closeEvent)); + } } // Y_UNIT_TEST_SUITE(TReadSessionDecompressionAccounting) } // namespace NYdb::inline V3::NTopic From 3545918ef66a758f587ab8d9f1ba4391fffcc6d1 Mon Sep 17 00:00:00 2001 From: kulaad Date: Wed, 11 Nov 2026 17:08:30 +0000 Subject: [PATCH 12/17] [WASM services 5/20] FQ ExternalService connections (#55858) --- .github/last_commit.txt | 2 +- src/api/protos/draft/fq.proto | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index f3f1f4f1132..bb5345b1d26 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -0962f20baf13702db98024d8f1206a2f0398e670 +6e3629b18daf4a2441d2b129fa3b1ae84564610f diff --git a/src/api/protos/draft/fq.proto b/src/api/protos/draft/fq.proto index 20f48783977..2f6543f349f 100644 --- a/src/api/protos/draft/fq.proto +++ b/src/api/protos/draft/fq.proto @@ -580,6 +580,27 @@ message Iceberg { IcebergCatalog catalog = 4; } +// Host-owned transport settings for external service adapters. +message ExternalService { + enum Protocol { + PROTOCOL_UNSPECIFIED = 0; + HTTP = 1; + GRPC = 2; + } + Protocol protocol = 1; + // HTTP origin (scheme + host + optional port) or gRPC host:port. Never supplied by WASM. + string endpoint = 2 [(Ydb.length).range = {min: 1, max: 4096}]; + // Methods and paths are supplied per invocation by WASM. + reserved 3; + reserved "method"; + IamAuth auth = 4; + // Explicit plaintext opt-in. TLS verification cannot be disabled. + bool insecure = 5; + // PEM trust roots; empty uses system roots. Not a server filesystem path. + string ca_certificate = 6 [(Ydb.length).le = 65536]; + map headers = 7 [(Ydb.sensitive) = true]; +} + message ConnectionSetting { enum ConnectionType { CONNECTION_TYPE_UNSPECIFIED = 0; @@ -593,6 +614,7 @@ message ConnectionSetting { MYSQL_CLUSTER = 8; LOGGING = 9; ICEBERG = 10; + EXTERNAL_SERVICE = 11; } oneof connection { @@ -606,6 +628,7 @@ message ConnectionSetting { MySQLCluster mysql_cluster = 8; Logging logging = 9; Iceberg iceberg = 10; + ExternalService external_service = 11; } } From aa475159a478abacbbb2e70fa5354315ce8b327a Mon Sep 17 00:00:00 2001 From: YDBot Date: Wed, 11 Nov 2026 17:08:38 +0000 Subject: [PATCH 13/17] Merge pull request #59028 from ydb-platform/robots/branch-sync/merge-rightlib-261111-0922 --- .github/last_commit.txt | 2 +- library/cpp/monlib/counters/counters.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index bb5345b1d26..f28eeeb3acb 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -6e3629b18daf4a2441d2b129fa3b1ae84564610f +a0d91898bc3c15b54ea46c204e8d8818e17ff53e diff --git a/library/cpp/monlib/counters/counters.h b/library/cpp/monlib/counters/counters.h index d3943d65787..4768dffb17e 100644 --- a/library/cpp/monlib/counters/counters.h +++ b/library/cpp/monlib/counters/counters.h @@ -104,7 +104,7 @@ namespace NMonitoring { } TValueBase operator=(TValueBase rhs) { - AtomicSwap(&Value, rhs); + Set(rhs); return rhs; } From 053cc954e12058ec56cc5ed673df3e9c43206097 Mon Sep 17 00:00:00 2001 From: YDBot Date: Wed, 11 Nov 2026 17:08:45 +0000 Subject: [PATCH 14/17] Mark C++ SDK v3.27.0 in changelog (#59093) --- .github/last_commit.txt | 2 +- CHANGELOG.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/last_commit.txt b/.github/last_commit.txt index f28eeeb3acb..a0829418a1c 100644 --- a/.github/last_commit.txt +++ b/.github/last_commit.txt @@ -1 +1 @@ -a0d91898bc3c15b54ea46c204e8d8818e17ff53e +9dcf0822149e3d827787f80bbf63dfa9245863c6 diff --git a/CHANGELOG.md b/CHANGELOG.md index 34bac278746..15d596d4bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v3.27.0 + * Fixed static login credentials failing permanently when the initial Login operation returns a transient service error. * Added draft `TBridgeClient::Rollback()` to roll a cluster converted into a bridged one back to ordinary mode, and `TBridgeClient::GetRollbackStatus()` to show the progress of the rollback. From 490113c6c08a26fd633491ccd9cca5c5b35f8adf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:08:45 +0000 Subject: [PATCH 15/17] Update import generation: 54 --- .github/import_generation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/import_generation.txt b/.github/import_generation.txt index fb1e7bc8699..c3f407c0955 100644 --- a/.github/import_generation.txt +++ b/.github/import_generation.txt @@ -1 +1 @@ -54 +55 From e46c8ae766a84d22260b38acce54bcb00ff2e1a8 Mon Sep 17 00:00:00 2001 From: YDBot Date: Wed, 11 Nov 2026 18:09:01 +0100 Subject: [PATCH 16/17] Release v3.27.0 --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index 24c1b713a36..7a4dda28498 100644 --- a/src/version.h +++ b/src/version.h @@ -2,7 +2,7 @@ namespace NYdb { -inline const char* YDB_SDK_VERSION = "3.26.0"; +inline const char* YDB_SDK_VERSION = "3.27.0"; inline const char* YDB_CERTIFICATE_FILE_KEY = "ydb_root_ca_v3.pem"; } // namespace NYdb From 83eea134fe292fe7d8358e07f54afd66bf3008ed Mon Sep 17 00:00:00 2001 From: Ermoshkin Artem <94714022+Shfdis@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:33:50 +0300 Subject: [PATCH 17/17] Fix static resource fallback linkage --- library/cpp/resource/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/library/cpp/resource/CMakeLists.txt b/library/cpp/resource/CMakeLists.txt index 3359da55794..a490d07a050 100644 --- a/library/cpp/resource/CMakeLists.txt +++ b/library/cpp/resource/CMakeLists.txt @@ -12,6 +12,7 @@ target_sources(resource PRIVATE registry.cpp resource.cpp + static_blob_default.cpp ) _ydb_sdk_install_targets(TARGETS resource)