From 571b11e5b125c41e2e5319b5c1fbfb9f354aab88 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Mon, 26 Oct 2026 18:57:58 +0300 Subject: [PATCH 1/7] Add topic attributes to Reader metrics --- .../ydb/topic/read/impl/BufferManager.java | 12 +- .../tech/ydb/topic/read/impl/ReadConfig.java | 1 + .../ydb/topic/read/impl/ReadPartition.java | 3 +- .../topic/read/impl/ReadPartitionQueue.java | 13 +- .../tech/ydb/topic/read/impl/ReadSession.java | 10 +- .../ydb/topic/read/impl/ReaderMetrics.java | 54 ++++--- .../ydb/topic/read/impl/SyncReaderImpl.java | 8 +- .../topic/read/impl/ReaderMetricsTest.java | 145 ++++++++++++++++-- 8 files changed, 199 insertions(+), 47 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java index 76d73d96e..f8b13eddb 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java @@ -7,6 +7,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; import java.util.function.Consumer; import org.slf4j.Logger; @@ -50,6 +51,11 @@ long getCreditBalanceBytes() { // Has no reentrant thread safety public void allocate(long bufferSize, List dataList) { + allocate(bufferSize, dataList, (partitionId, bytes) -> { }); + } + + void allocate(long bufferSize, List dataList, + BiConsumer allocated) { logger.debug("[{}] Received ReadResponse of {} bytes, {} allocated and {} released before", traceID, bufferSize, totalAllocated.get(), totalReleased.get()); totalAllocated.addAndGet(bufferSize); @@ -87,6 +93,7 @@ public void allocate(long bufferSize, List metrics.addTopic(topic.getPath())); } public CodecRegistry getCodecRegistry() { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index 483bb8897..cc02aba86 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -77,7 +77,8 @@ private enum State { this.committer = new ReadPartitionCommitter(traceID, session, partition, lastCommittedOffset); this.decoder = new ReadPartitionDecoder(traceID, sessionDecoder, partition, committer, this::sendDataToReaders, metrics); - this.queue = new ReadPartitionQueue(traceID, decoder, config.getMaxBatchSize(), lastCommittedOffset); + this.queue = new ReadPartitionQueue(traceID, partition.getPath(), decoder, + config.getMaxBatchSize(), lastCommittedOffset); logger.info("[{}] started for {}", traceID, partition); } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java index 0b3755c64..a3cfda6a5 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionQueue.java @@ -20,13 +20,16 @@ class ReadPartitionQueue implements ReaderMetrics.Source { private static final Logger logger = LoggerFactory.getLogger(ReadPartition.class); private final String traceID; + private final String topic; private final ReadPartitionDecoder decoder; private final int maxBatchSize; private final Queue readingQueue = new ConcurrentLinkedQueue<>(); private volatile long lastReadOffset; - ReadPartitionQueue(String traceID, ReadPartitionDecoder decoder, int maxBatchSize, long lastReadOffset) { + ReadPartitionQueue(String traceID, String topic, ReadPartitionDecoder decoder, int maxBatchSize, + long lastReadOffset) { this.traceID = traceID; + this.topic = topic; this.decoder = decoder; this.maxBatchSize = maxBatchSize; this.lastReadOffset = lastReadOffset; @@ -37,13 +40,13 @@ void updateLastReadOffset(long offset) { } @Override - public long getMessagesCount() { - return readingQueue.size(); + public long getMessagesCount(String topic) { + return this.topic.equals(topic) ? readingQueue.size() : 0; } @Override - public long getOldestReceivedTimestamp() { - MessageImpl first = readingQueue.peek(); + public long getOldestReceivedTimestamp(String topic) { + MessageImpl first = this.topic.equals(topic) ? readingQueue.peek() : null; return first == null ? 0 : first.getReceivedAt(); } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index 4c53d2e94..4aa56f34d 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -224,7 +224,7 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } - config.getMetrics().addPartition(psid, committed); + config.getMetrics().addPartition(psid, committed, ps.getPath()); partition.start(committed, offsets); TopicPartition lockKey = new TopicPartition(ps); if (locks.computeIfPresent(lockKey, (key, lock) -> lock.bind(partition)) != null) { @@ -286,8 +286,12 @@ public void onStopPartition(YdbTopic.StreamReadMessage.StopPartitionSessionReque public void onRead(YdbTopic.StreamReadMessage.ReadResponse response) { logger.debug("[{}] Received ReadResponse of {} bytes", debugId, response.getBytesSize()); - config.getMetrics().reportReceivedBytes(response.getBytesSize()); - bufferManager.allocate(response.getBytesSize(), response.getPartitionDataList()); + bufferManager.allocate(response.getBytesSize(), response.getPartitionDataList(), (partitionId, bytes) -> { + ReadPartition partition = partitions.get(partitionId); + if (partition != null) { + config.getMetrics().reportReceivedBytes(bytes, partition.getPartition().getPath()); + } + }); for (YdbTopic.StreamReadMessage.ReadResponse.PartitionData data: response.getPartitionDataList()) { long psid = data.getPartitionSessionId(); diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java index 72e9ac192..27055db40 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java @@ -21,8 +21,8 @@ */ final class ReaderMetrics { interface Source { - long getMessagesCount(); - long getOldestReceivedTimestamp(); + long getMessagesCount(String topic); + long getOldestReceivedTimestamp(String topic); } private static final String MESSAGE_UNIT = "{message}"; @@ -40,6 +40,7 @@ interface Source { private final Meter meter; private final Set sources = ConcurrentHashMap.newKeySet(); private final Map partitions = new ConcurrentHashMap<>(); + private final Map topics = new ConcurrentHashMap<>(); private MetricRegistration partitionsGauge = MetricRegistration.NOOP; private MetricRegistration creditGauge = MetricRegistration.NOOP; private MetricRegistration commitLagGauge = MetricRegistration.NOOP; @@ -78,8 +79,17 @@ void removeSource(Source source) { sources.remove(source); } - void addPartition(long id, long committedOffset) { - partitions.put(id, new CommitOffsets(committedOffset)); + void addTopic(String topic) { + topics.computeIfAbsent(topic, name -> { + Attr[] attributes = Arrays.copyOf(commonAttributes, commonAttributes.length + 1); + attributes[commonAttributes.length] = Attr.of("topic", name); + return attributes; + }); + } + + void addPartition(long id, long committedOffset, String topic) { + addTopic(topic); + partitions.put(id, new CommitOffsets(committedOffset, topic)); } void removePartition(long id) { @@ -93,19 +103,20 @@ void updateCommittedOffset(long id, long offset) { } } - private long getCommitOffsetLagMax() { + private long getCommitOffsetLagMax(String topic) { return partitions.values().stream() + .filter(partition -> topic.equals(partition.topic)) .mapToLong(p -> Math.max(0, p.requested.get() - p.committed.get())).max().orElse(0); } - private long getLocalBufferMessages() { - return sources.stream().mapToLong(Source::getMessagesCount).sum(); + private long getLocalBufferMessages(String topic) { + return sources.stream().mapToLong(source -> source.getMessagesCount(topic)).sum(); } - private long getOldestReceivedTimestamp() { + private long getOldestReceivedTimestamp(String topic) { long oldest = 0; for (Source source : sources) { - long timestamp = source.getOldestReceivedTimestamp(); + long timestamp = source.getOldestReceivedTimestamp(topic); if (timestamp != 0 && (oldest == 0 || timestamp < oldest)) { oldest = timestamp; } @@ -117,23 +128,24 @@ void register(LongSupplier bufferBudget) { partitionsGauge = meter.registerLongGauge( "ydb.topic.reader.partition_session.count", "{session}", "The number of partition sessions currently in the reader session processing lifecycle.", - m -> m.record(partitions.size(), commonAttributes)); + m -> topics.forEach((topic, attributes) -> m.record(partitions.values().stream() + .filter(partition -> topic.equals(partition.topic)).count(), attributes))); creditGauge = meter.registerLongGauge("ydb.topic.reader.credit_balance_bytes", "By", "The protocol credit granted to the server and not yet consumed by read responses.", m -> m.record(bufferBudget.getAsLong(), commonAttributes)); commitLagGauge = meter.registerLongGauge("ydb.topic.reader.commit_offset.lag.max", MESSAGE_UNIT, "The maximum gap between requested and acknowledged commit offsets.", - m -> m.record(getCommitOffsetLagMax(), commonAttributes)); + m -> topics.forEach((topic, attributes) -> m.record(getCommitOffsetLagMax(topic), attributes))); bufferMessagesGauge = meter.registerLongGauge("ydb.topic.reader.local_buffer.messages", MESSAGE_UNIT, "The number of messages currently buffered by the reader.", - m -> m.record(getLocalBufferMessages(), commonAttributes)); + m -> topics.forEach((topic, attributes) -> m.record(getLocalBufferMessages(topic), attributes))); ageGauge = meter.registerDoubleGauge("ydb.topic.reader.local_buffer.message_age.max", "s", "The age of the oldest batch retained in the reader's local buffer.", - m -> { - long timestamp = getOldestReceivedTimestamp(); + m -> topics.forEach((topic, attributes) -> { + long timestamp = getOldestReceivedTimestamp(topic); m.record(timestamp == 0 ? 0 : (System.nanoTime() - timestamp) / 1_000_000_000d, - commonAttributes); - }); + attributes); + })); } void unregister() { @@ -159,10 +171,8 @@ void reportReceivedMessages(long messages, String topic) { report(receivedMessages, messages, topic); } - void reportReceivedBytes(long bytes) { - if (enabled) { - record(receivedBytes, bytes); - } + void reportReceivedBytes(long bytes, String topic) { + report(receivedBytes, bytes, topic); } void reportSessionError(Status status, boolean retry) { @@ -240,10 +250,12 @@ private static Attr[] createCommonAttributes(String consumer, String readerName) } private static class CommitOffsets { + private final String topic; private final AtomicLong requested; private final AtomicLong committed; - CommitOffsets(long offset) { + CommitOffsets(long offset, String topic) { + this.topic = topic; this.requested = new AtomicLong(offset); this.committed = new AtomicLong(offset); } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java index 984c997da..c740a5340 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java @@ -225,13 +225,15 @@ public void onSessionStarted(String sessionId) { SyncReaderImpl.this.sessionId = sessionId; config.getMetrics().addSource(new ReaderMetrics.Source() { @Override - public long getMessagesCount() { - return queue.size(); + public long getMessagesCount(String topic) { + return queue.stream() + .filter(message -> topic.equals(message.msg.getPartitionSession().getPath())).count(); } @Override - public long getOldestReceivedTimestamp() { + public long getOldestReceivedTimestamp(String topic) { return queue.stream().filter(MessageWrapper::isActive) + .filter(message -> topic.equals(message.msg.getPartitionSession().getPath())) .mapToLong(message -> ((MessageImpl) message.msg).getReceivedAt()).min().orElse(0); } }); diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index a70f57ee8..60df8e5a5 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -2,7 +2,11 @@ import java.io.IOException; import java.io.InputStream; +import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Deque; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; @@ -19,9 +23,11 @@ import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; import tech.ydb.topic.impl.TopicClientImpl; +import tech.ydb.topic.read.AsyncReader; import tech.ydb.topic.read.DecompressionException; import tech.ydb.topic.read.Message; import tech.ydb.topic.read.SyncReader; +import tech.ydb.topic.settings.ReadEventHandlersSettings; import tech.ydb.topic.settings.ReaderSettings; import tech.ydb.topic.utils.InMemoryMeter; @@ -50,9 +56,17 @@ public void readAndCommitMetrics() throws Exception { reader.init(); stream.assertLastMessage().isInitRequest("consumer", "/topic", "/other-topic").hasReaderName("reader"); stream.responseInit("read-session"); + for (Map topic : Arrays.asList(TOPIC, OTHER)) { + meter.assertLongGauge(PREFIX + "partition_session.count", 0, topic); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, topic); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, topic); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, topic); + } stream.responseStartPartition("/topic", 42, 0); stream.responseStartPartition("/other-topic", 43, 0); - meter.assertLongGauge(PREFIX + "partition_session.count", 2, COMMON); + stream.responseStartPartition("/topic", 44, 0); + meter.assertLongGauge(PREFIX + "partition_session.count", 2, TOPIC); + meter.assertLongGauge(PREFIX + "partition_session.count", 1, OTHER); meter.assertLongGauge(PREFIX + "credit_balance_bytes", 100, COMMON); stream.responseData(12).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); meter.assertCounter(PREFIX + "received.messages", 2, TOPIC); @@ -60,14 +74,23 @@ public void readAndCommitMetrics() throws Exception { stream.responseData(8).partition(2, 0).batch(Codec.GZIP, MSG1).and().send(); meter.assertCounter(PREFIX + "received.messages", 2, TOPIC); meter.assertCounter(PREFIX + "received.messages", 1, OTHER); - meter.assertCounter(PREFIX + "received.bytes", 20, COMMON); + meter.assertCounter(PREFIX + "received.bytes", 12, TOPIC); + meter.assertCounter(PREFIX + "received.bytes", 8, OTHER); meter.assertLongGauge(PREFIX + "credit_balance_bytes", 80, COMMON); meter.assertCounter(PREFIX + "delivered.messages", 0, TOPIC); meter.assertCounter(PREFIX + "delivered.messages", 0, OTHER); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 3, COMMON); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 2, TOPIC); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", TOPIC); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", OTHER); Message first = reader.receive(0, TimeUnit.SECONDS); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, TOPIC); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); Message second = reader.receive(0, TimeUnit.SECONDS); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, TOPIC); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, TOPIC); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", OTHER); Message third = reader.receive(0, TimeUnit.SECONDS); Assert.assertNotNull(first); Assert.assertNotNull(second); @@ -77,8 +100,10 @@ public void readAndCommitMetrics() throws Exception { Assert.assertArrayEquals(MSG1, third.getData()); meter.assertCounter(PREFIX + "delivered.messages", 2, TOPIC); meter.assertCounter(PREFIX + "delivered.messages", 1, OTHER); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, COMMON); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, TOPIC); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, OTHER); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, TOPIC); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, OTHER); // The final 8 released bytes stay below the 10-byte credit-request threshold. meter.assertLongGauge(PREFIX + "credit_balance_bytes", 92, COMMON); CompletableFuture committed = CompletableFuture.allOf( @@ -87,12 +112,18 @@ public void readAndCommitMetrics() throws Exception { meter.assertCounter(PREFIX + "commit.queued", 2, TOPIC); meter.assertCounter(PREFIX + "commit.queued", 1, OTHER); meter.assertNoCounterSamples(PREFIX + "commit.acknowledged"); - meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 2, COMMON); - stream.responseCommitAck().partition(1, 2).partition(2, 1).send(); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 2, TOPIC); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 1, OTHER); + stream.responseCommitAck().partition(1, 2).send(); + Assert.assertFalse(committed.isDone()); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, TOPIC); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 1, OTHER); + stream.responseCommitAck().partition(2, 1).send(); committed.get(1, TimeUnit.SECONDS); meter.assertCounter(PREFIX + "commit.acknowledged", 2, TOPIC); meter.assertCounter(PREFIX + "commit.acknowledged", 1, OTHER); - meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, COMMON); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, TOPIC); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, OTHER); for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { meter.assertHistogram(PREFIX + name, "s", 2, attributes("consumer", "consumer", "reader.name", "reader", "topic", "/topic", "codec", "gzip", "outcome", "ok")); @@ -101,8 +132,12 @@ public void readAndCommitMetrics() throws Exception { } meter.assertUnit(PREFIX + "received.bytes", "By"); stream.responseStopPartition(1, true); + meter.assertLongGauge(PREFIX + "partition_session.count", 1, TOPIC); + meter.assertLongGauge(PREFIX + "partition_session.count", 1, OTHER); stream.responseStopPartition(2, true); - meter.assertLongGauge(PREFIX + "partition_session.count", 0, COMMON); + stream.responseStopPartition(3, true); + meter.assertLongGauge(PREFIX + "partition_session.count", 0, TOPIC); + meter.assertLongGauge(PREFIX + "partition_session.count", 0, OTHER); } finally { reader.shutdown(); } @@ -127,13 +162,13 @@ public void readWithoutConsumerMetrics() throws Exception { stream.responseStartPartition("/topic", 42, 0); stream.responseData(8).partition(1, 0).batch(Codec.GZIP, MSG1).and().send(); meter.assertCounter(PREFIX + "received.messages", 1, topic); - meter.assertCounter(PREFIX + "received.bytes", 8, common); + meter.assertCounter(PREFIX + "received.bytes", 8, topic); Message message = reader.receive(0, TimeUnit.SECONDS); Assert.assertNotNull(message); Assert.assertArrayEquals(MSG1, message.getData()); meter.assertCounter(PREFIX + "delivered.messages", 1, topic); - meter.assertLongGauge(PREFIX + "partition_session.count", 1, common); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, common); + meter.assertLongGauge(PREFIX + "partition_session.count", 1, topic); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, topic); for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { meter.assertHistogram(PREFIX + name, "s", 1, attributes("reader.name", "without-consumer", "topic", "/topic", "codec", "gzip", "outcome", "ok")); @@ -218,6 +253,90 @@ public void decompressionFailureMetrics() throws Exception { meter.assertNoGauges(); } + @Test + public void receivedBytesFollowPartitionAllocations() throws Exception { + ReadStreamMock stream = new ReadStreamMock(true); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).build()) { + SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder() + .addTopic("/topic").addTopic("/other-topic").setConsumerName("consumer") + .setDecompressionExecutor(Runnable::run).withMeter(meter, "reader").build()); + try { + reader.init(); + stream.responseInit("mixed-response"); + stream.responseStartPartition("/topic", 42, 0); + stream.responseStartPartition("/other-topic", 43, 0); + stream.responseData(25).partition(1, 0).batch(Codec.RAW, MSG1).and() + .partition(2, 0).batch(Codec.RAW, MSG2).and().send(); + meter.assertCounter(PREFIX + "received.bytes", 10, TOPIC); + meter.assertCounter(PREFIX + "received.bytes", 15, OTHER); + meter.assertCounterSeries(PREFIX + "received.bytes", 2); + stream.responseData(5).partition(1, 1).batch(Codec.RAW, new byte[0]).and() + .partition(2, 1).batch(Codec.RAW, new byte[0]).and().send(); + meter.assertCounter(PREFIX + "received.bytes", 10, TOPIC); + meter.assertCounter(PREFIX + "received.bytes", 20, OTHER); + Assert.assertArrayEquals(MSG1, reader.receive(0, TimeUnit.SECONDS).getData()); + Assert.assertArrayEquals(MSG2, reader.receive(0, TimeUnit.SECONDS).getData()); + Assert.assertArrayEquals(new byte[0], reader.receive(0, TimeUnit.SECONDS).getData()); + Assert.assertArrayEquals(new byte[0], reader.receive(0, TimeUnit.SECONDS).getData()); + } finally { + reader.shutdown(); + } + } + meter.assertNoGauges(); + } + + @Test + public void asyncBuffersKeepTopicOwnership() throws Exception { + ReadStreamMock stream = new ReadStreamMock(true); + Deque tasks = new ArrayDeque<>(); + List delivered = new ArrayList<>(); + try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).build()) { + AsyncReader reader = client.createAsyncReader(ReaderSettings.newBuilder() + .addTopic("/topic").addTopic("/other-topic").setConsumerName("consumer") + .setDecompressionExecutor(Runnable::run).withMeter(meter, "reader").build(), + ReadEventHandlersSettings.newBuilder().setExecutor(tasks::add) + .setEventHandler(event -> delivered.addAll(event.getMessages())).build()); + try { + reader.init(); + stream.responseInit("async-buffers"); + stream.responseStartPartition("/topic", 42, 0); + stream.responseStartPartition("/other-topic", 43, 0); + while (!tasks.isEmpty()) { + tasks.remove().run(); + } + stream.responseData(12).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); + stream.responseData(8).partition(2, 0).batch(Codec.GZIP, MSG1).and().send(); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 2, TOPIC); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", TOPIC); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", OTHER); + tasks.remove().run(); + Assert.assertEquals(2, delivered.size()); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, TOPIC); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, TOPIC); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); + while (!tasks.isEmpty()) { + tasks.remove().run(); + } + Assert.assertEquals(3, delivered.size()); + Assert.assertArrayEquals(MSG1, delivered.get(0).getData()); + Assert.assertArrayEquals(MSG2, delivered.get(1).getData()); + Assert.assertArrayEquals(MSG1, delivered.get(2).getData()); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, OTHER); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, OTHER); + meter.assertCounter(PREFIX + "delivered.messages", 2, TOPIC); + meter.assertCounter(PREFIX + "delivered.messages", 1, OTHER); + } finally { + CompletableFuture closed = reader.shutdown(); + while (!tasks.isEmpty()) { + tasks.remove().run(); + } + closed.get(1, TimeUnit.SECONDS); + } + } + meter.assertNoGauges(); + } + private static TopicRpc mockRpc(ReadStreamMock first, ReadStreamMock... rest) { TopicRpc rpc = Mockito.mock(TopicRpc.class); Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); From 5ee09fb6daa835f1f24700447c357b5e9604cd0c Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Tue, 27 Oct 2026 09:54:22 +0300 Subject: [PATCH 2/7] Keep Reader buffer and byte metrics session-wide --- .../ydb/topic/read/impl/BufferManager.java | 12 +--- .../ydb/topic/read/impl/ReadPartition.java | 3 +- .../topic/read/impl/ReadPartitionQueue.java | 13 ++-- .../tech/ydb/topic/read/impl/ReadSession.java | 8 +-- .../ydb/topic/read/impl/ReaderMetrics.java | 28 +++++---- .../ydb/topic/read/impl/SyncReaderImpl.java | 8 +-- .../topic/read/impl/ReaderMetricsTest.java | 61 ++++++++----------- 7 files changed, 52 insertions(+), 81 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java index f8b13eddb..76d73d96e 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/BufferManager.java @@ -7,7 +7,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.BiConsumer; import java.util.function.Consumer; import org.slf4j.Logger; @@ -51,11 +50,6 @@ long getCreditBalanceBytes() { // Has no reentrant thread safety public void allocate(long bufferSize, List dataList) { - allocate(bufferSize, dataList, (partitionId, bytes) -> { }); - } - - void allocate(long bufferSize, List dataList, - BiConsumer allocated) { logger.debug("[{}] Received ReadResponse of {} bytes, {} allocated and {} released before", traceID, bufferSize, totalAllocated.get(), totalReleased.get()); totalAllocated.addAndGet(bufferSize); @@ -93,7 +87,6 @@ void allocate(long bufferSize, List readingQueue = new ConcurrentLinkedQueue<>(); private volatile long lastReadOffset; - ReadPartitionQueue(String traceID, String topic, ReadPartitionDecoder decoder, int maxBatchSize, - long lastReadOffset) { + ReadPartitionQueue(String traceID, ReadPartitionDecoder decoder, int maxBatchSize, long lastReadOffset) { this.traceID = traceID; - this.topic = topic; this.decoder = decoder; this.maxBatchSize = maxBatchSize; this.lastReadOffset = lastReadOffset; @@ -40,13 +37,13 @@ void updateLastReadOffset(long offset) { } @Override - public long getMessagesCount(String topic) { - return this.topic.equals(topic) ? readingQueue.size() : 0; + public long getMessagesCount() { + return readingQueue.size(); } @Override - public long getOldestReceivedTimestamp(String topic) { - MessageImpl first = this.topic.equals(topic) ? readingQueue.peek() : null; + public long getOldestReceivedTimestamp() { + MessageImpl first = readingQueue.peek(); return first == null ? 0 : first.getReceivedAt(); } diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index 4aa56f34d..80656609c 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -286,12 +286,8 @@ public void onStopPartition(YdbTopic.StreamReadMessage.StopPartitionSessionReque public void onRead(YdbTopic.StreamReadMessage.ReadResponse response) { logger.debug("[{}] Received ReadResponse of {} bytes", debugId, response.getBytesSize()); - bufferManager.allocate(response.getBytesSize(), response.getPartitionDataList(), (partitionId, bytes) -> { - ReadPartition partition = partitions.get(partitionId); - if (partition != null) { - config.getMetrics().reportReceivedBytes(bytes, partition.getPartition().getPath()); - } - }); + config.getMetrics().reportReceivedBytes(response.getBytesSize()); + bufferManager.allocate(response.getBytesSize(), response.getPartitionDataList()); for (YdbTopic.StreamReadMessage.ReadResponse.PartitionData data: response.getPartitionDataList()) { long psid = data.getPartitionSessionId(); diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java index 27055db40..cc8056c4a 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java @@ -21,8 +21,8 @@ */ final class ReaderMetrics { interface Source { - long getMessagesCount(String topic); - long getOldestReceivedTimestamp(String topic); + long getMessagesCount(); + long getOldestReceivedTimestamp(); } private static final String MESSAGE_UNIT = "{message}"; @@ -109,14 +109,14 @@ private long getCommitOffsetLagMax(String topic) { .mapToLong(p -> Math.max(0, p.requested.get() - p.committed.get())).max().orElse(0); } - private long getLocalBufferMessages(String topic) { - return sources.stream().mapToLong(source -> source.getMessagesCount(topic)).sum(); + private long getLocalBufferMessages() { + return sources.stream().mapToLong(Source::getMessagesCount).sum(); } - private long getOldestReceivedTimestamp(String topic) { + private long getOldestReceivedTimestamp() { long oldest = 0; for (Source source : sources) { - long timestamp = source.getOldestReceivedTimestamp(topic); + long timestamp = source.getOldestReceivedTimestamp(); if (timestamp != 0 && (oldest == 0 || timestamp < oldest)) { oldest = timestamp; } @@ -138,14 +138,14 @@ void register(LongSupplier bufferBudget) { m -> topics.forEach((topic, attributes) -> m.record(getCommitOffsetLagMax(topic), attributes))); bufferMessagesGauge = meter.registerLongGauge("ydb.topic.reader.local_buffer.messages", MESSAGE_UNIT, "The number of messages currently buffered by the reader.", - m -> topics.forEach((topic, attributes) -> m.record(getLocalBufferMessages(topic), attributes))); + m -> m.record(getLocalBufferMessages(), commonAttributes)); ageGauge = meter.registerDoubleGauge("ydb.topic.reader.local_buffer.message_age.max", "s", "The age of the oldest batch retained in the reader's local buffer.", - m -> topics.forEach((topic, attributes) -> { - long timestamp = getOldestReceivedTimestamp(topic); + m -> { + long timestamp = getOldestReceivedTimestamp(); m.record(timestamp == 0 ? 0 : (System.nanoTime() - timestamp) / 1_000_000_000d, - attributes); - })); + commonAttributes); + }); } void unregister() { @@ -171,8 +171,10 @@ void reportReceivedMessages(long messages, String topic) { report(receivedMessages, messages, topic); } - void reportReceivedBytes(long bytes, String topic) { - report(receivedBytes, bytes, topic); + void reportReceivedBytes(long bytes) { + if (enabled) { + record(receivedBytes, bytes); + } } void reportSessionError(Status status, boolean retry) { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java index c740a5340..984c997da 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/SyncReaderImpl.java @@ -225,15 +225,13 @@ public void onSessionStarted(String sessionId) { SyncReaderImpl.this.sessionId = sessionId; config.getMetrics().addSource(new ReaderMetrics.Source() { @Override - public long getMessagesCount(String topic) { - return queue.stream() - .filter(message -> topic.equals(message.msg.getPartitionSession().getPath())).count(); + public long getMessagesCount() { + return queue.size(); } @Override - public long getOldestReceivedTimestamp(String topic) { + public long getOldestReceivedTimestamp() { return queue.stream().filter(MessageWrapper::isActive) - .filter(message -> topic.equals(message.msg.getPartitionSession().getPath())) .mapToLong(message -> ((MessageImpl) message.msg).getReceivedAt()).min().orElse(0); } }); diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index 60df8e5a5..c95da1a1d 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -58,10 +58,10 @@ public void readAndCommitMetrics() throws Exception { stream.responseInit("read-session"); for (Map topic : Arrays.asList(TOPIC, OTHER)) { meter.assertLongGauge(PREFIX + "partition_session.count", 0, topic); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, topic); meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, topic); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, topic); } + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, COMMON); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); stream.responseStartPartition("/topic", 42, 0); stream.responseStartPartition("/other-topic", 43, 0); stream.responseStartPartition("/topic", 44, 0); @@ -74,23 +74,18 @@ public void readAndCommitMetrics() throws Exception { stream.responseData(8).partition(2, 0).batch(Codec.GZIP, MSG1).and().send(); meter.assertCounter(PREFIX + "received.messages", 2, TOPIC); meter.assertCounter(PREFIX + "received.messages", 1, OTHER); - meter.assertCounter(PREFIX + "received.bytes", 12, TOPIC); - meter.assertCounter(PREFIX + "received.bytes", 8, OTHER); + meter.assertCounter(PREFIX + "received.bytes", 20, COMMON); + meter.assertCounterSeries(PREFIX + "received.bytes", 1); meter.assertLongGauge(PREFIX + "credit_balance_bytes", 80, COMMON); meter.assertCounter(PREFIX + "delivered.messages", 0, TOPIC); meter.assertCounter(PREFIX + "delivered.messages", 0, OTHER); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 2, TOPIC); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", TOPIC); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 3, COMMON); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); Message first = reader.receive(0, TimeUnit.SECONDS); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, TOPIC); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 2, COMMON); Message second = reader.receive(0, TimeUnit.SECONDS); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, TOPIC); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, TOPIC); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, COMMON); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); Message third = reader.receive(0, TimeUnit.SECONDS); Assert.assertNotNull(first); Assert.assertNotNull(second); @@ -100,10 +95,8 @@ public void readAndCommitMetrics() throws Exception { Assert.assertArrayEquals(MSG1, third.getData()); meter.assertCounter(PREFIX + "delivered.messages", 2, TOPIC); meter.assertCounter(PREFIX + "delivered.messages", 1, OTHER); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, TOPIC); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, OTHER); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, TOPIC); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, COMMON); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); // The final 8 released bytes stay below the 10-byte credit-request threshold. meter.assertLongGauge(PREFIX + "credit_balance_bytes", 92, COMMON); CompletableFuture committed = CompletableFuture.allOf( @@ -162,13 +155,13 @@ public void readWithoutConsumerMetrics() throws Exception { stream.responseStartPartition("/topic", 42, 0); stream.responseData(8).partition(1, 0).batch(Codec.GZIP, MSG1).and().send(); meter.assertCounter(PREFIX + "received.messages", 1, topic); - meter.assertCounter(PREFIX + "received.bytes", 8, topic); + meter.assertCounter(PREFIX + "received.bytes", 8, common); Message message = reader.receive(0, TimeUnit.SECONDS); Assert.assertNotNull(message); Assert.assertArrayEquals(MSG1, message.getData()); meter.assertCounter(PREFIX + "delivered.messages", 1, topic); meter.assertLongGauge(PREFIX + "partition_session.count", 1, topic); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, topic); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, common); for (String name : Arrays.asList("compression.duration", "message.processing.duration")) { meter.assertHistogram(PREFIX + name, "s", 1, attributes("reader.name", "without-consumer", "topic", "/topic", "codec", "gzip", "outcome", "ok")); @@ -254,7 +247,7 @@ public void decompressionFailureMetrics() throws Exception { } @Test - public void receivedBytesFollowPartitionAllocations() throws Exception { + public void receivedBytesAreSharedAcrossTopics() throws Exception { ReadStreamMock stream = new ReadStreamMock(true); try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).build()) { SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder() @@ -267,13 +260,12 @@ public void receivedBytesFollowPartitionAllocations() throws Exception { stream.responseStartPartition("/other-topic", 43, 0); stream.responseData(25).partition(1, 0).batch(Codec.RAW, MSG1).and() .partition(2, 0).batch(Codec.RAW, MSG2).and().send(); - meter.assertCounter(PREFIX + "received.bytes", 10, TOPIC); - meter.assertCounter(PREFIX + "received.bytes", 15, OTHER); - meter.assertCounterSeries(PREFIX + "received.bytes", 2); + meter.assertCounter(PREFIX + "received.bytes", 25, COMMON); + meter.assertCounterSeries(PREFIX + "received.bytes", 1); stream.responseData(5).partition(1, 1).batch(Codec.RAW, new byte[0]).and() .partition(2, 1).batch(Codec.RAW, new byte[0]).and().send(); - meter.assertCounter(PREFIX + "received.bytes", 10, TOPIC); - meter.assertCounter(PREFIX + "received.bytes", 20, OTHER); + meter.assertCounter(PREFIX + "received.bytes", 30, COMMON); + meter.assertCounterSeries(PREFIX + "received.bytes", 1); Assert.assertArrayEquals(MSG1, reader.receive(0, TimeUnit.SECONDS).getData()); Assert.assertArrayEquals(MSG2, reader.receive(0, TimeUnit.SECONDS).getData()); Assert.assertArrayEquals(new byte[0], reader.receive(0, TimeUnit.SECONDS).getData()); @@ -286,7 +278,7 @@ public void receivedBytesFollowPartitionAllocations() throws Exception { } @Test - public void asyncBuffersKeepTopicOwnership() throws Exception { + public void asyncBuffersAreSharedAcrossTopics() throws Exception { ReadStreamMock stream = new ReadStreamMock(true); Deque tasks = new ArrayDeque<>(); List delivered = new ArrayList<>(); @@ -306,15 +298,12 @@ public void asyncBuffersKeepTopicOwnership() throws Exception { } stream.responseData(12).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); stream.responseData(8).partition(2, 0).batch(Codec.GZIP, MSG1).and().send(); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 2, TOPIC); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", TOPIC); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 3, COMMON); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); tasks.remove().run(); Assert.assertEquals(2, delivered.size()); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, TOPIC); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, TOPIC); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, COMMON); + meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); while (!tasks.isEmpty()) { tasks.remove().run(); } @@ -322,8 +311,8 @@ public void asyncBuffersKeepTopicOwnership() throws Exception { Assert.assertArrayEquals(MSG1, delivered.get(0).getData()); Assert.assertArrayEquals(MSG2, delivered.get(1).getData()); Assert.assertArrayEquals(MSG1, delivered.get(2).getData()); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, OTHER); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, OTHER); + meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, COMMON); + meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); meter.assertCounter(PREFIX + "delivered.messages", 2, TOPIC); meter.assertCounter(PREFIX + "delivered.messages", 1, OTHER); } finally { From 1c43ada059e47218a23d0f951e972056c1ce2c5c Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Tue, 27 Oct 2026 10:49:07 +0300 Subject: [PATCH 3/7] Collect per-topic Reader gauges in ReadSession --- .../tech/ydb/topic/read/impl/ReadConfig.java | 1 - .../tech/ydb/topic/read/impl/ReadSession.java | 75 ++++++++++++++++--- .../ydb/topic/read/impl/ReaderMetrics.java | 70 +++-------------- .../topic/read/impl/ReaderMetricsTest.java | 6 +- 4 files changed, 82 insertions(+), 70 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadConfig.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadConfig.java index 44a572957..b01a84fbe 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadConfig.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadConfig.java @@ -29,7 +29,6 @@ public ReadConfig(CodecRegistry codecRegistry, Executor control, Executor proces this.maxMemoryUsageBytes = settings.getMaxMemoryUsageBytes(); this.maxBatchSize = settings.getMaxBatchSize(); this.metrics = new ReaderMetrics(settings.getMeter(), settings.getConsumerName(), settings.getReaderName()); - settings.getTopics().forEach(topic -> metrics.addTopic(topic.getPath())); } public CodecRegistry getCodecRegistry() { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index 80656609c..672dc9e7b 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -1,11 +1,13 @@ package tech.ydb.topic.read.impl; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -59,8 +61,10 @@ public interface Handler { private final MessageDecoder decoder; private final BufferManager bufferManager; private final Handler handler; + private final YdbTopic.StreamReadMessage.InitRequest initRequest; private final Map partitions = new ConcurrentHashMap<>(); + private final Map commits = new ConcurrentHashMap<>(); private final Map locks = new ConcurrentHashMap<>(); private volatile boolean isClosed = false; @@ -72,6 +76,7 @@ public ReadSession(String id, GrpcReadWriteStream stream this.decoder = new MessageDecoder(config); this.bufferManager = new BufferManager(id, config.getMaxMemoryUsageBytes(), new ReadRequest()); this.handler = handler; + this.initRequest = initReq.getInitRequest(); } @Override @@ -108,6 +113,7 @@ protected void preDestroy() { locks.clear(); partitions.values().forEach(ReadPartition::close); partitions.clear(); + commits.clear(); } public boolean commitOffsets(PartitionSession session, List rangesToCommit) { @@ -129,8 +135,11 @@ public boolean commitOffsets(PartitionSession session, List ranges .build(); for (OffsetsRange range : rangesToCommit) { - config.getMetrics().reportCommitQueued(session.getId(), range.getEnd(), - range.getEnd() - range.getStart(), session.getPath()); + CommitOffsets offsets = commits.get(session.getId()); + if (offsets != null) { + offsets.requested.accumulateAndGet(range.getEnd(), Math::max); + } + config.getMetrics().reportCommitQueued(range.getEnd() - range.getStart(), session.getPath()); } send(FromClient.newBuilder().setCommitOffsetRequest(req).build()); return true; @@ -149,7 +158,7 @@ public void sendStartPartition(PartitionSession ps, Long readFrom, Long commitTo } if (commitTo != null) { resp.setCommitOffset(commitTo); - config.getMetrics().updateCommittedOffset(ps.getId(), commitTo); + updateCommittedOffset(ps.getId(), commitTo); } send(YdbTopic.StreamReadMessage.FromClient.newBuilder().setStartPartitionSessionResponse(resp.build()).build()); @@ -169,7 +178,7 @@ public void sendStopPartition(PartitionSession ps) { ReadPartition partition = partitions.remove(ps.getId()); if (partition != null) { - config.getMetrics().removePartition(ps.getId()); + commits.remove(ps.getId()); partition.close(); } } @@ -195,7 +204,38 @@ public void removeLocks(PartitionSession ps, List childs) { public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { bufferManager.init(response.getSessionId()); - config.getMetrics().register(bufferManager::getCreditBalanceBytes); + config.getMetrics().register(bufferManager::getCreditBalanceBytes, + this::getPartitionCounts, this::getCommitOffsetLagMax); + } + + private Map newTopicValues() { + Map values = new HashMap<>(); + initRequest.getTopicsReadSettingsList().forEach(topic -> values.put(topic.getPath(), 0L)); + return values; + } + + private Map getPartitionCounts() { + Map counts = newTopicValues(); + partitions.values().forEach(partition -> + counts.merge(partition.getPartition().getPath(), 1L, Long::sum)); + return counts; + } + + private Map getCommitOffsetLagMax() { + Map lag = newTopicValues(); + partitions.forEach((id, partition) -> { + CommitOffsets offsets = commits.get(id); + long value = offsets == null ? 0 : Math.max(0, offsets.requested.get() - offsets.committed.get()); + lag.merge(partition.getPartition().getPath(), value, Math::max); + }); + return lag; + } + + private void updateCommittedOffset(long id, long offset) { + CommitOffsets offsets = commits.get(id); + if (offsets != null) { + offsets.committed.accumulateAndGet(offset, Math::max); + } } public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionRequest req) { @@ -224,7 +264,7 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } - config.getMetrics().addPartition(psid, committed, ps.getPath()); + commits.put(psid, new CommitOffsets(committed)); partition.start(committed, offsets); TopicPartition lockKey = new TopicPartition(ps); if (locks.computeIfPresent(lockKey, (key, lock) -> lock.bind(partition)) != null) { @@ -263,7 +303,7 @@ public void onClosePartition(long psid) { } logger.info("[{}] Received force StopPartitionSessionRequest for {} ", debugId, partition.getPartition()); - config.getMetrics().removePartition(psid); + commits.remove(psid); partition.close(); } @@ -272,7 +312,7 @@ public void onStopPartition(YdbTopic.StreamReadMessage.StopPartitionSessionReque long psid = request.getPartitionSessionId(); ReadPartition partition = partitions.get(psid); if (partition != null) { - config.getMetrics().updateCommittedOffset(psid, committedOffset); + updateCommittedOffset(psid, committedOffset); partition.stop(committedOffset); return; } @@ -313,8 +353,13 @@ public void onCommitOffset(YdbTopic.StreamReadMessage.CommitOffsetResponse respo continue; } - config.getMetrics().reportCommitAcknowledged(offset.getPartitionSessionId(), - offset.getCommittedOffset(), partition.getPartition().getPath()); + CommitOffsets offsets = commits.get(offset.getPartitionSessionId()); + if (offsets != null) { + long acknowledged = offset.getCommittedOffset(); + long old = offsets.committed.getAndAccumulate(acknowledged, Math::max); + config.getMetrics().reportCommitAcknowledged( + acknowledged > old ? acknowledged - old : 0, partition.getPartition().getPath()); + } partition.confirmCommittedOffset(offset.getCommittedOffset()); } } @@ -330,6 +375,16 @@ public void onPartitionSessionStatus(YdbTopic.StreamReadMessage.PartitionSession resp.getCommittedOffset()); } + private static class CommitOffsets { + private final AtomicLong requested; + private final AtomicLong committed; + + CommitOffsets(long offset) { + this.requested = new AtomicLong(offset); + this.committed = new AtomicLong(offset); + } + } + private class ReadRequest implements Consumer { @Override public void accept(Long sizeToRequest) { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java index cc8056c4a..db0fa200c 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java @@ -4,8 +4,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongSupplier; +import java.util.function.Supplier; import tech.ydb.core.Status; import tech.ydb.core.StatusCode; @@ -39,8 +39,6 @@ interface Source { private final boolean enabled; private final Meter meter; private final Set sources = ConcurrentHashMap.newKeySet(); - private final Map partitions = new ConcurrentHashMap<>(); - private final Map topics = new ConcurrentHashMap<>(); private MetricRegistration partitionsGauge = MetricRegistration.NOOP; private MetricRegistration creditGauge = MetricRegistration.NOOP; private MetricRegistration commitLagGauge = MetricRegistration.NOOP; @@ -79,34 +77,10 @@ void removeSource(Source source) { sources.remove(source); } - void addTopic(String topic) { - topics.computeIfAbsent(topic, name -> { - Attr[] attributes = Arrays.copyOf(commonAttributes, commonAttributes.length + 1); - attributes[commonAttributes.length] = Attr.of("topic", name); - return attributes; - }); - } - - void addPartition(long id, long committedOffset, String topic) { - addTopic(topic); - partitions.put(id, new CommitOffsets(committedOffset, topic)); - } - - void removePartition(long id) { - partitions.remove(id); - } - - void updateCommittedOffset(long id, long offset) { - CommitOffsets offsets = partitions.get(id); - if (offsets != null) { - offsets.committed.accumulateAndGet(offset, Math::max); - } - } - - private long getCommitOffsetLagMax(String topic) { - return partitions.values().stream() - .filter(partition -> topic.equals(partition.topic)) - .mapToLong(p -> Math.max(0, p.requested.get() - p.committed.get())).max().orElse(0); + private Attr[] topicAttributes(String topic) { + Attr[] attributes = Arrays.copyOf(commonAttributes, commonAttributes.length + 1); + attributes[commonAttributes.length] = Attr.of("topic", topic); + return attributes; } private long getLocalBufferMessages() { @@ -124,18 +98,18 @@ private long getOldestReceivedTimestamp() { return oldest; } - void register(LongSupplier bufferBudget) { + void register(LongSupplier bufferBudget, Supplier> partitionCounts, + Supplier> commitLag) { partitionsGauge = meter.registerLongGauge( "ydb.topic.reader.partition_session.count", "{session}", "The number of partition sessions currently in the reader session processing lifecycle.", - m -> topics.forEach((topic, attributes) -> m.record(partitions.values().stream() - .filter(partition -> topic.equals(partition.topic)).count(), attributes))); + m -> partitionCounts.get().forEach((topic, count) -> m.record(count, topicAttributes(topic)))); creditGauge = meter.registerLongGauge("ydb.topic.reader.credit_balance_bytes", "By", "The protocol credit granted to the server and not yet consumed by read responses.", m -> m.record(bufferBudget.getAsLong(), commonAttributes)); commitLagGauge = meter.registerLongGauge("ydb.topic.reader.commit_offset.lag.max", MESSAGE_UNIT, "The maximum gap between requested and acknowledged commit offsets.", - m -> topics.forEach((topic, attributes) -> m.record(getCommitOffsetLagMax(topic), attributes))); + m -> commitLag.get().forEach((topic, lag) -> m.record(lag, topicAttributes(topic)))); bufferMessagesGauge = meter.registerLongGauge("ydb.topic.reader.local_buffer.messages", MESSAGE_UNIT, "The number of messages currently buffered by the reader.", m -> m.record(getLocalBufferMessages(), commonAttributes)); @@ -160,7 +134,6 @@ void unregister() { bufferMessagesGauge = MetricRegistration.NOOP; ageGauge = MetricRegistration.NOOP; sources.clear(); - partitions.clear(); } void reportDelivered(long messages, String topic) { @@ -189,20 +162,12 @@ void reportSessionError(Status status, boolean retry) { sessionErrors.add(1, attributes); } - void reportCommitQueued(long partitionId, long offset, long messages, String topic) { - CommitOffsets offsets = partitions.get(partitionId); - if (offsets != null) { - offsets.requested.accumulateAndGet(offset, Math::max); - } + void reportCommitQueued(long messages, String topic) { report(commitQueued, messages, topic); } - void reportCommitAcknowledged(long partitionId, long offset, String topic) { - CommitOffsets offsets = partitions.get(partitionId); - if (offsets != null) { - long old = offsets.committed.getAndAccumulate(offset, Math::max); - report(commitAcknowledged, offset > old ? offset - old : 0, topic); - } + void reportCommitAcknowledged(long messages, String topic) { + report(commitAcknowledged, messages, topic); } void reportDecoded(long compressionNanos, long processingNanos, int codec, String topic, Throwable error) { @@ -251,15 +216,4 @@ private static Attr[] createCommonAttributes(String consumer, String readerName) return new Attr[]{Attr.of("consumer", consumer), Attr.of("reader.name", readerName)}; } - private static class CommitOffsets { - private final String topic; - private final AtomicLong requested; - private final AtomicLong committed; - - CommitOffsets(long offset, String topic) { - this.topic = topic; - this.requested = new AtomicLong(offset); - this.committed = new AtomicLong(offset); - } - } } diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index c95da1a1d..a687f73ec 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -99,10 +99,14 @@ public void readAndCommitMetrics() throws Exception { meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); // The final 8 released bytes stay below the 10-byte credit-request threshold. meter.assertLongGauge(PREFIX + "credit_balance_bytes", 92, COMMON); + first.getCommitter().commitRanges(Arrays.asList( + first.getRangeToCommit(), second.getRangeToCommit())); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 2, TOPIC); + meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, OTHER); CompletableFuture committed = CompletableFuture.allOf( first.commit(), second.commit(), third.commit()); Assert.assertFalse(committed.isDone()); - meter.assertCounter(PREFIX + "commit.queued", 2, TOPIC); + meter.assertCounter(PREFIX + "commit.queued", 4, TOPIC); meter.assertCounter(PREFIX + "commit.queued", 1, OTHER); meter.assertNoCounterSamples(PREFIX + "commit.acknowledged"); meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 2, TOPIC); From f404cb4e256a9fbe6d8608b77160c034d0a3b0c6 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Tue, 27 Oct 2026 11:21:34 +0300 Subject: [PATCH 4/7] Group Reader metrics by existing session partitions --- .../tech/ydb/topic/read/impl/ReadSession.java | 75 +++---------------- .../ydb/topic/read/impl/ReaderMetrics.java | 61 +++++++++++++-- .../topic/read/impl/ReaderMetricsTest.java | 10 +-- .../tech/ydb/topic/utils/InMemoryMeter.java | 11 +++ 4 files changed, 78 insertions(+), 79 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index 672dc9e7b..994a5aafb 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -1,13 +1,11 @@ package tech.ydb.topic.read.impl; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -61,10 +59,8 @@ public interface Handler { private final MessageDecoder decoder; private final BufferManager bufferManager; private final Handler handler; - private final YdbTopic.StreamReadMessage.InitRequest initRequest; private final Map partitions = new ConcurrentHashMap<>(); - private final Map commits = new ConcurrentHashMap<>(); private final Map locks = new ConcurrentHashMap<>(); private volatile boolean isClosed = false; @@ -76,7 +72,6 @@ public ReadSession(String id, GrpcReadWriteStream stream this.decoder = new MessageDecoder(config); this.bufferManager = new BufferManager(id, config.getMaxMemoryUsageBytes(), new ReadRequest()); this.handler = handler; - this.initRequest = initReq.getInitRequest(); } @Override @@ -113,7 +108,6 @@ protected void preDestroy() { locks.clear(); partitions.values().forEach(ReadPartition::close); partitions.clear(); - commits.clear(); } public boolean commitOffsets(PartitionSession session, List rangesToCommit) { @@ -135,11 +129,8 @@ public boolean commitOffsets(PartitionSession session, List ranges .build(); for (OffsetsRange range : rangesToCommit) { - CommitOffsets offsets = commits.get(session.getId()); - if (offsets != null) { - offsets.requested.accumulateAndGet(range.getEnd(), Math::max); - } - config.getMetrics().reportCommitQueued(range.getEnd() - range.getStart(), session.getPath()); + config.getMetrics().reportCommitQueued(session.getId(), range.getEnd(), + range.getEnd() - range.getStart(), session.getPath()); } send(FromClient.newBuilder().setCommitOffsetRequest(req).build()); return true; @@ -158,7 +149,7 @@ public void sendStartPartition(PartitionSession ps, Long readFrom, Long commitTo } if (commitTo != null) { resp.setCommitOffset(commitTo); - updateCommittedOffset(ps.getId(), commitTo); + config.getMetrics().updateCommittedOffset(ps.getId(), commitTo); } send(YdbTopic.StreamReadMessage.FromClient.newBuilder().setStartPartitionSessionResponse(resp.build()).build()); @@ -178,7 +169,7 @@ public void sendStopPartition(PartitionSession ps) { ReadPartition partition = partitions.remove(ps.getId()); if (partition != null) { - commits.remove(ps.getId()); + config.getMetrics().removePartition(ps.getId()); partition.close(); } } @@ -204,38 +195,7 @@ public void removeLocks(PartitionSession ps, List childs) { public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { bufferManager.init(response.getSessionId()); - config.getMetrics().register(bufferManager::getCreditBalanceBytes, - this::getPartitionCounts, this::getCommitOffsetLagMax); - } - - private Map newTopicValues() { - Map values = new HashMap<>(); - initRequest.getTopicsReadSettingsList().forEach(topic -> values.put(topic.getPath(), 0L)); - return values; - } - - private Map getPartitionCounts() { - Map counts = newTopicValues(); - partitions.values().forEach(partition -> - counts.merge(partition.getPartition().getPath(), 1L, Long::sum)); - return counts; - } - - private Map getCommitOffsetLagMax() { - Map lag = newTopicValues(); - partitions.forEach((id, partition) -> { - CommitOffsets offsets = commits.get(id); - long value = offsets == null ? 0 : Math.max(0, offsets.requested.get() - offsets.committed.get()); - lag.merge(partition.getPartition().getPath(), value, Math::max); - }); - return lag; - } - - private void updateCommittedOffset(long id, long offset) { - CommitOffsets offsets = commits.get(id); - if (offsets != null) { - offsets.committed.accumulateAndGet(offset, Math::max); - } + config.getMetrics().register(bufferManager::getCreditBalanceBytes, partitions.values()); } public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionRequest req) { @@ -264,7 +224,7 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } - commits.put(psid, new CommitOffsets(committed)); + config.getMetrics().addPartition(psid, committed); partition.start(committed, offsets); TopicPartition lockKey = new TopicPartition(ps); if (locks.computeIfPresent(lockKey, (key, lock) -> lock.bind(partition)) != null) { @@ -303,7 +263,7 @@ public void onClosePartition(long psid) { } logger.info("[{}] Received force StopPartitionSessionRequest for {} ", debugId, partition.getPartition()); - commits.remove(psid); + config.getMetrics().removePartition(psid); partition.close(); } @@ -312,7 +272,7 @@ public void onStopPartition(YdbTopic.StreamReadMessage.StopPartitionSessionReque long psid = request.getPartitionSessionId(); ReadPartition partition = partitions.get(psid); if (partition != null) { - updateCommittedOffset(psid, committedOffset); + config.getMetrics().updateCommittedOffset(psid, committedOffset); partition.stop(committedOffset); return; } @@ -353,13 +313,8 @@ public void onCommitOffset(YdbTopic.StreamReadMessage.CommitOffsetResponse respo continue; } - CommitOffsets offsets = commits.get(offset.getPartitionSessionId()); - if (offsets != null) { - long acknowledged = offset.getCommittedOffset(); - long old = offsets.committed.getAndAccumulate(acknowledged, Math::max); - config.getMetrics().reportCommitAcknowledged( - acknowledged > old ? acknowledged - old : 0, partition.getPartition().getPath()); - } + config.getMetrics().reportCommitAcknowledged(offset.getPartitionSessionId(), + offset.getCommittedOffset(), partition.getPartition().getPath()); partition.confirmCommittedOffset(offset.getCommittedOffset()); } } @@ -375,16 +330,6 @@ public void onPartitionSessionStatus(YdbTopic.StreamReadMessage.PartitionSession resp.getCommittedOffset()); } - private static class CommitOffsets { - private final AtomicLong requested; - private final AtomicLong committed; - - CommitOffsets(long offset) { - this.requested = new AtomicLong(offset); - this.committed = new AtomicLong(offset); - } - } - private class ReadRequest implements Consumer { @Override public void accept(Long sizeToRequest) { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java index db0fa200c..2ec16249d 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java @@ -1,11 +1,13 @@ package tech.ydb.topic.read.impl; import java.util.Arrays; +import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongSupplier; -import java.util.function.Supplier; +import java.util.stream.Collectors; import tech.ydb.core.Status; import tech.ydb.core.StatusCode; @@ -39,6 +41,7 @@ interface Source { private final boolean enabled; private final Meter meter; private final Set sources = ConcurrentHashMap.newKeySet(); + private final Map partitions = new ConcurrentHashMap<>(); private MetricRegistration partitionsGauge = MetricRegistration.NOOP; private MetricRegistration creditGauge = MetricRegistration.NOOP; private MetricRegistration commitLagGauge = MetricRegistration.NOOP; @@ -77,6 +80,26 @@ void removeSource(Source source) { sources.remove(source); } + void addPartition(long id, long committedOffset) { + partitions.put(id, new CommitOffsets(committedOffset)); + } + + void removePartition(long id) { + partitions.remove(id); + } + + void updateCommittedOffset(long id, long offset) { + CommitOffsets offsets = partitions.get(id); + if (offsets != null) { + offsets.committed.accumulateAndGet(offset, Math::max); + } + } + + private long getCommitOffsetLag(long id) { + CommitOffsets offsets = partitions.get(id); + return offsets == null ? 0 : Math.max(0, offsets.requested.get() - offsets.committed.get()); + } + private Attr[] topicAttributes(String topic) { Attr[] attributes = Arrays.copyOf(commonAttributes, commonAttributes.length + 1); attributes[commonAttributes.length] = Attr.of("topic", topic); @@ -98,18 +121,22 @@ private long getOldestReceivedTimestamp() { return oldest; } - void register(LongSupplier bufferBudget, Supplier> partitionCounts, - Supplier> commitLag) { + void register(LongSupplier bufferBudget, Collection readPartitions) { partitionsGauge = meter.registerLongGauge( "ydb.topic.reader.partition_session.count", "{session}", "The number of partition sessions currently in the reader session processing lifecycle.", - m -> partitionCounts.get().forEach((topic, count) -> m.record(count, topicAttributes(topic)))); + m -> readPartitions.stream().collect(Collectors.groupingBy( + partition -> partition.getPartition().getPath(), Collectors.counting())) + .forEach((topic, count) -> m.record(count, topicAttributes(topic)))); creditGauge = meter.registerLongGauge("ydb.topic.reader.credit_balance_bytes", "By", "The protocol credit granted to the server and not yet consumed by read responses.", m -> m.record(bufferBudget.getAsLong(), commonAttributes)); commitLagGauge = meter.registerLongGauge("ydb.topic.reader.commit_offset.lag.max", MESSAGE_UNIT, "The maximum gap between requested and acknowledged commit offsets.", - m -> commitLag.get().forEach((topic, lag) -> m.record(lag, topicAttributes(topic)))); + m -> readPartitions.stream().collect(Collectors.toMap( + partition -> partition.getPartition().getPath(), + partition -> getCommitOffsetLag(partition.getPartition().getId()), Math::max)) + .forEach((topic, lag) -> m.record(lag, topicAttributes(topic)))); bufferMessagesGauge = meter.registerLongGauge("ydb.topic.reader.local_buffer.messages", MESSAGE_UNIT, "The number of messages currently buffered by the reader.", m -> m.record(getLocalBufferMessages(), commonAttributes)); @@ -134,6 +161,7 @@ void unregister() { bufferMessagesGauge = MetricRegistration.NOOP; ageGauge = MetricRegistration.NOOP; sources.clear(); + partitions.clear(); } void reportDelivered(long messages, String topic) { @@ -162,12 +190,20 @@ void reportSessionError(Status status, boolean retry) { sessionErrors.add(1, attributes); } - void reportCommitQueued(long messages, String topic) { + void reportCommitQueued(long partitionId, long offset, long messages, String topic) { + CommitOffsets offsets = partitions.get(partitionId); + if (offsets != null) { + offsets.requested.accumulateAndGet(offset, Math::max); + } report(commitQueued, messages, topic); } - void reportCommitAcknowledged(long messages, String topic) { - report(commitAcknowledged, messages, topic); + void reportCommitAcknowledged(long partitionId, long offset, String topic) { + CommitOffsets offsets = partitions.get(partitionId); + if (offsets != null) { + long old = offsets.committed.getAndAccumulate(offset, Math::max); + report(commitAcknowledged, offset > old ? offset - old : 0, topic); + } } void reportDecoded(long compressionNanos, long processingNanos, int codec, String topic, Throwable error) { @@ -216,4 +252,13 @@ private static Attr[] createCommonAttributes(String consumer, String readerName) return new Attr[]{Attr.of("consumer", consumer), Attr.of("reader.name", readerName)}; } + private static class CommitOffsets { + private final AtomicLong requested; + private final AtomicLong committed; + + CommitOffsets(long offset) { + this.requested = new AtomicLong(offset); + this.committed = new AtomicLong(offset); + } + } } diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index a687f73ec..72d302edf 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -56,10 +56,8 @@ public void readAndCommitMetrics() throws Exception { reader.init(); stream.assertLastMessage().isInitRequest("consumer", "/topic", "/other-topic").hasReaderName("reader"); stream.responseInit("read-session"); - for (Map topic : Arrays.asList(TOPIC, OTHER)) { - meter.assertLongGauge(PREFIX + "partition_session.count", 0, topic); - meter.assertLongGauge(PREFIX + "commit_offset.lag.max", 0, topic); - } + meter.assertLongGaugeSeries(PREFIX + "partition_session.count", 0); + meter.assertLongGaugeSeries(PREFIX + "commit_offset.lag.max", 0); meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, COMMON); meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); stream.responseStartPartition("/topic", 42, 0); @@ -133,8 +131,8 @@ public void readAndCommitMetrics() throws Exception { meter.assertLongGauge(PREFIX + "partition_session.count", 1, OTHER); stream.responseStopPartition(2, true); stream.responseStopPartition(3, true); - meter.assertLongGauge(PREFIX + "partition_session.count", 0, TOPIC); - meter.assertLongGauge(PREFIX + "partition_session.count", 0, OTHER); + meter.assertLongGaugeSeries(PREFIX + "partition_session.count", 0); + meter.assertLongGaugeSeries(PREFIX + "commit_offset.lag.max", 0); } finally { reader.shutdown(); } diff --git a/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java b/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java index 8cbbfea14..435cab90e 100644 --- a/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java +++ b/topic/src/test/java/tech/ydb/topic/utils/InMemoryMeter.java @@ -103,6 +103,17 @@ public void assertLongGauge(String name, long expected, Map attr Assert.assertEquals(name + " " + attributes, expected, values.get(attributes).longValue()); } + public void assertLongGaugeSeries(String name, int expected) { + List> callbacks = longGauges.get(name); + Assert.assertNotNull(name, callbacks); + Map, Long> values = new HashMap<>(); + for (Consumer callback : callbacks) { + callback.accept((value, attrs) -> Assert.assertNull(name + " duplicate series", + values.put(key(attrs), value))); + } + Assert.assertEquals(name, expected, values.size()); + } + public void assertDoubleGauge(String name, double expected, Map attributes) { Assert.assertEquals(name + " " + attributes, expected, doubleGaugeValue(name, attributes), 0); } From db7b144e808a69b23f23b89940070632fb20865d Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Tue, 27 Oct 2026 12:17:11 +0300 Subject: [PATCH 5/7] Remove redundant AsyncReader metrics test --- .../topic/read/impl/ReaderMetricsTest.java | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index 72d302edf..cdba969ca 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -2,11 +2,7 @@ import java.io.IOException; import java.io.InputStream; -import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Deque; -import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; @@ -23,11 +19,9 @@ import tech.ydb.topic.TopicRpc; import tech.ydb.topic.description.Codec; import tech.ydb.topic.impl.TopicClientImpl; -import tech.ydb.topic.read.AsyncReader; import tech.ydb.topic.read.DecompressionException; import tech.ydb.topic.read.Message; import tech.ydb.topic.read.SyncReader; -import tech.ydb.topic.settings.ReadEventHandlersSettings; import tech.ydb.topic.settings.ReaderSettings; import tech.ydb.topic.utils.InMemoryMeter; @@ -279,55 +273,6 @@ public void receivedBytesAreSharedAcrossTopics() throws Exception { meter.assertNoGauges(); } - @Test - public void asyncBuffersAreSharedAcrossTopics() throws Exception { - ReadStreamMock stream = new ReadStreamMock(true); - Deque tasks = new ArrayDeque<>(); - List delivered = new ArrayList<>(); - try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).build()) { - AsyncReader reader = client.createAsyncReader(ReaderSettings.newBuilder() - .addTopic("/topic").addTopic("/other-topic").setConsumerName("consumer") - .setDecompressionExecutor(Runnable::run).withMeter(meter, "reader").build(), - ReadEventHandlersSettings.newBuilder().setExecutor(tasks::add) - .setEventHandler(event -> delivered.addAll(event.getMessages())).build()); - try { - reader.init(); - stream.responseInit("async-buffers"); - stream.responseStartPartition("/topic", 42, 0); - stream.responseStartPartition("/other-topic", 43, 0); - while (!tasks.isEmpty()) { - tasks.remove().run(); - } - stream.responseData(12).partition(1, 0).batch(Codec.GZIP, MSG1, MSG2).and().send(); - stream.responseData(8).partition(2, 0).batch(Codec.GZIP, MSG1).and().send(); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 3, COMMON); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); - tasks.remove().run(); - Assert.assertEquals(2, delivered.size()); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 1, COMMON); - meter.assertPositiveDoubleGauge(PREFIX + "local_buffer.message_age.max", COMMON); - while (!tasks.isEmpty()) { - tasks.remove().run(); - } - Assert.assertEquals(3, delivered.size()); - Assert.assertArrayEquals(MSG1, delivered.get(0).getData()); - Assert.assertArrayEquals(MSG2, delivered.get(1).getData()); - Assert.assertArrayEquals(MSG1, delivered.get(2).getData()); - meter.assertLongGauge(PREFIX + "local_buffer.messages", 0, COMMON); - meter.assertDoubleGauge(PREFIX + "local_buffer.message_age.max", 0, COMMON); - meter.assertCounter(PREFIX + "delivered.messages", 2, TOPIC); - meter.assertCounter(PREFIX + "delivered.messages", 1, OTHER); - } finally { - CompletableFuture closed = reader.shutdown(); - while (!tasks.isEmpty()) { - tasks.remove().run(); - } - closed.get(1, TimeUnit.SECONDS); - } - } - meter.assertNoGauges(); - } - private static TopicRpc mockRpc(ReadStreamMock first, ReadStreamMock... rest) { TopicRpc rpc = Mockito.mock(TopicRpc.class); Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class)); From 247d0141386bae68d083e0493e9ccab1bec47993 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Tue, 27 Oct 2026 13:38:17 +0300 Subject: [PATCH 6/7] Keep topic paths in Reader metric partition state --- .../tech/ydb/topic/read/impl/ReadSession.java | 4 +-- .../ydb/topic/read/impl/ReaderMetrics.java | 26 ++++++++----------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index 994a5aafb..80656609c 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -195,7 +195,7 @@ public void removeLocks(PartitionSession ps, List childs) { public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { bufferManager.init(response.getSessionId()); - config.getMetrics().register(bufferManager::getCreditBalanceBytes, partitions.values()); + config.getMetrics().register(bufferManager::getCreditBalanceBytes); } public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionRequest req) { @@ -224,7 +224,7 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } - config.getMetrics().addPartition(psid, committed); + config.getMetrics().addPartition(psid, committed, ps.getPath()); partition.start(committed, offsets); TopicPartition lockKey = new TopicPartition(ps); if (locks.computeIfPresent(lockKey, (key, lock) -> lock.bind(partition)) != null) { diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java index 2ec16249d..d4e9f1edf 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java @@ -1,7 +1,6 @@ package tech.ydb.topic.read.impl; import java.util.Arrays; -import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -80,8 +79,8 @@ void removeSource(Source source) { sources.remove(source); } - void addPartition(long id, long committedOffset) { - partitions.put(id, new CommitOffsets(committedOffset)); + void addPartition(long id, long committedOffset, String topicPath) { + partitions.put(id, new CommitOffsets(committedOffset, topicPath)); } void removePartition(long id) { @@ -95,11 +94,6 @@ void updateCommittedOffset(long id, long offset) { } } - private long getCommitOffsetLag(long id) { - CommitOffsets offsets = partitions.get(id); - return offsets == null ? 0 : Math.max(0, offsets.requested.get() - offsets.committed.get()); - } - private Attr[] topicAttributes(String topic) { Attr[] attributes = Arrays.copyOf(commonAttributes, commonAttributes.length + 1); attributes[commonAttributes.length] = Attr.of("topic", topic); @@ -121,21 +115,21 @@ private long getOldestReceivedTimestamp() { return oldest; } - void register(LongSupplier bufferBudget, Collection readPartitions) { + void register(LongSupplier bufferBudget) { partitionsGauge = meter.registerLongGauge( "ydb.topic.reader.partition_session.count", "{session}", "The number of partition sessions currently in the reader session processing lifecycle.", - m -> readPartitions.stream().collect(Collectors.groupingBy( - partition -> partition.getPartition().getPath(), Collectors.counting())) + m -> partitions.values().stream().collect(Collectors.groupingBy( + offsets -> offsets.topicPath, Collectors.counting())) .forEach((topic, count) -> m.record(count, topicAttributes(topic)))); creditGauge = meter.registerLongGauge("ydb.topic.reader.credit_balance_bytes", "By", "The protocol credit granted to the server and not yet consumed by read responses.", m -> m.record(bufferBudget.getAsLong(), commonAttributes)); commitLagGauge = meter.registerLongGauge("ydb.topic.reader.commit_offset.lag.max", MESSAGE_UNIT, "The maximum gap between requested and acknowledged commit offsets.", - m -> readPartitions.stream().collect(Collectors.toMap( - partition -> partition.getPartition().getPath(), - partition -> getCommitOffsetLag(partition.getPartition().getId()), Math::max)) + m -> partitions.values().stream().collect(Collectors.toMap( + offsets -> offsets.topicPath, + offsets -> Math.max(0, offsets.requested.get() - offsets.committed.get()), Math::max)) .forEach((topic, lag) -> m.record(lag, topicAttributes(topic)))); bufferMessagesGauge = meter.registerLongGauge("ydb.topic.reader.local_buffer.messages", MESSAGE_UNIT, "The number of messages currently buffered by the reader.", @@ -253,10 +247,12 @@ private static Attr[] createCommonAttributes(String consumer, String readerName) } private static class CommitOffsets { + private final String topicPath; private final AtomicLong requested; private final AtomicLong committed; - CommitOffsets(long offset) { + CommitOffsets(long offset, String topicPath) { + this.topicPath = topicPath; this.requested = new AtomicLong(offset); this.committed = new AtomicLong(offset); } From c749e2b50842747aa4f21ea9b1c6bfc7cf8c2a86 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Tue, 27 Oct 2026 13:48:35 +0300 Subject: [PATCH 7/7] Remove redundant shared received bytes test --- .../topic/read/impl/ReaderMetricsTest.java | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java index cdba969ca..99a27b725 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/ReaderMetricsTest.java @@ -242,37 +242,6 @@ public void decompressionFailureMetrics() throws Exception { meter.assertNoGauges(); } - @Test - public void receivedBytesAreSharedAcrossTopics() throws Exception { - ReadStreamMock stream = new ReadStreamMock(true); - try (TopicClient client = TopicClientImpl.newClient(mockRpc(stream)).build()) { - SyncReader reader = client.createSyncReader(ReaderSettings.newBuilder() - .addTopic("/topic").addTopic("/other-topic").setConsumerName("consumer") - .setDecompressionExecutor(Runnable::run).withMeter(meter, "reader").build()); - try { - reader.init(); - stream.responseInit("mixed-response"); - stream.responseStartPartition("/topic", 42, 0); - stream.responseStartPartition("/other-topic", 43, 0); - stream.responseData(25).partition(1, 0).batch(Codec.RAW, MSG1).and() - .partition(2, 0).batch(Codec.RAW, MSG2).and().send(); - meter.assertCounter(PREFIX + "received.bytes", 25, COMMON); - meter.assertCounterSeries(PREFIX + "received.bytes", 1); - stream.responseData(5).partition(1, 1).batch(Codec.RAW, new byte[0]).and() - .partition(2, 1).batch(Codec.RAW, new byte[0]).and().send(); - meter.assertCounter(PREFIX + "received.bytes", 30, COMMON); - meter.assertCounterSeries(PREFIX + "received.bytes", 1); - Assert.assertArrayEquals(MSG1, reader.receive(0, TimeUnit.SECONDS).getData()); - Assert.assertArrayEquals(MSG2, reader.receive(0, TimeUnit.SECONDS).getData()); - Assert.assertArrayEquals(new byte[0], reader.receive(0, TimeUnit.SECONDS).getData()); - Assert.assertArrayEquals(new byte[0], reader.receive(0, TimeUnit.SECONDS).getData()); - } finally { - reader.shutdown(); - } - } - meter.assertNoGauges(); - } - private static TopicRpc mockRpc(ReadStreamMock first, ReadStreamMock... rest) { TopicRpc rpc = Mockito.mock(TopicRpc.class); Mockito.when(rpc.getScheduler()).thenReturn(Mockito.mock(ScheduledExecutorService.class));