From 205c097eefbebc77cca2ef4a92a58f7012117276 Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Tue, 27 Oct 2026 11:14:14 +0000 Subject: [PATCH 1/2] Added WriteConfig class --- .../ydb/topic/write/impl/BufferManager.java | 7 +- .../ydb/topic/write/impl/WriteConfig.java | 50 ++++ .../tech/ydb/topic/write/impl/WriterImpl.java | 14 +- .../ydb/topic/write/impl/WriterQueue.java | 28 +- .../topic/write/impl/BufferManagerTest.java | 38 +-- .../ydb/topic/write/impl/WriterQueueTest.java | 275 +++++++++--------- 6 files changed, 224 insertions(+), 188 deletions(-) create mode 100644 topic/src/main/java/tech/ydb/topic/write/impl/WriteConfig.java diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java b/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java index d8a4a21a4..848978de7 100644 --- a/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java +++ b/topic/src/main/java/tech/ydb/topic/write/impl/BufferManager.java @@ -8,7 +8,6 @@ import org.slf4j.LoggerFactory; import tech.ydb.core.Status; -import tech.ydb.topic.settings.WriterSettings; import tech.ydb.topic.write.QueueOverflowException; /** @@ -30,11 +29,11 @@ public class BufferManager { private volatile Status closed = null; - public BufferManager(String id, WriterSettings settings) { + public BufferManager(String id, long bufferMaxSize, int bufferMaxMessageCount) { this.debugId = id; - this.maxCount = settings.getMaxSendBufferMessagesCount(); - this.bufferMaxSize = settings.getMaxSendBufferMemorySize(); + this.maxCount = bufferMaxMessageCount; + this.bufferMaxSize = bufferMaxSize; this.blockBitsCount = calculateBlockSize(bufferMaxSize); diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/WriteConfig.java b/topic/src/main/java/tech/ydb/topic/write/impl/WriteConfig.java new file mode 100644 index 000000000..4fa026a47 --- /dev/null +++ b/topic/src/main/java/tech/ydb/topic/write/impl/WriteConfig.java @@ -0,0 +1,50 @@ +package tech.ydb.topic.write.impl; + +import java.util.concurrent.Executor; + +import tech.ydb.topic.description.Codec; +import tech.ydb.topic.description.CodecRegistry; +import tech.ydb.topic.settings.WriterSettings; + +/** + * + * @author Aleksandr Gorshenin {@literal } + */ +public class WriteConfig { + private final long bufferMaxSize; + private final int bufferMaxMessagesCount; + private final Codec codec; + private final Executor compressor; + private final WriterMetrics metrics; + + public WriteConfig(CodecRegistry codecRegistry, Executor compressor, WriterSettings settings) { + this.bufferMaxSize = settings.getMaxSendBufferMemorySize(); + this.bufferMaxMessagesCount = settings.getMaxSendBufferMessagesCount(); + this.compressor = compressor; + this.metrics = new WriterMetrics(settings.getMeter(), settings.getTopicPath(), settings.getWriterName()); + this.codec = codecRegistry.getCodec(settings.getCodec()); + if (codec == null) { + throw new IllegalArgumentException("Unsupported codec: " + settings.getCodec()); + } + } + + public long getBufferMaxSize() { + return bufferMaxSize; + } + + public int getBufferMaxMessagesCount() { + return bufferMaxMessagesCount; + } + + public Executor getCompressor() { + return compressor; + } + + public Codec getCodec() { + return codec; + } + + WriterMetrics getMetrics() { + return metrics; + } +} diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/WriterImpl.java b/topic/src/main/java/tech/ydb/topic/write/impl/WriterImpl.java index 744e98a44..9c7992c13 100644 --- a/topic/src/main/java/tech/ydb/topic/write/impl/WriterImpl.java +++ b/topic/src/main/java/tech/ydb/topic/write/impl/WriterImpl.java @@ -39,7 +39,6 @@ public class WriterImpl { private final String debugId; private final WriterQueue writeQueue; - private final WriterMetrics metrics; private final WriteSession stream; private final Runnable sendTask = new SerialRunnable(new SendTask()); @@ -58,11 +57,12 @@ public WriterImpl(TopicRpc topicRpc, WriterSettings settings, Executor compressi public WriterImpl(TopicRpc topicRpc, WriteStreamFactory factory, WriterSettings settings, Executor compressionExecutor, @Nonnull CodecRegistry codecRegistry) { this.debugId = DebugTools.createDebugId(settings.getLogPrefix()); - this.metrics = new WriterMetrics(settings.getMeter(), settings.getTopicPath(), settings.getWriterName()); - this.stream = new WriteSession(debugId, factory, settings, topicRpc.getScheduler(), new ListenerImpl()); - this.writeQueue = new WriterQueue(debugId, settings, metrics, codecRegistry, compressionExecutor, sendTask); + + WriteConfig config = new WriteConfig(codecRegistry, compressionExecutor, settings); Observability.reportMetricsUsage(settings.getMeter()); + this.stream = new WriteSession(debugId, factory, settings, topicRpc.getScheduler(), new ListenerImpl(config)); + this.writeQueue = new WriterQueue(debugId, config, sendTask); logger.info("Writer with id {} created for topic \"{}\" with producerId \"{}\" and messageGroupId \"{}\"", debugId, settings.getTopicPath(), settings.getProducerId(), settings.getMessageGroupId()); } @@ -174,6 +174,12 @@ public CompletableFuture nonblockingSend(Message msg, SendSettings set } private class ListenerImpl implements WriteSession.Listener { + private final WriterMetrics metrics; + + ListenerImpl(WriteConfig config) { + this.metrics = config.getMetrics(); + } + @Override public void onStart(long lastSeqNo, String sessionId) { // resend all sent messages in writing queue diff --git a/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java b/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java index 1daf22d17..f2163111b 100644 --- a/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java +++ b/topic/src/main/java/tech/ydb/topic/write/impl/WriterQueue.java @@ -13,7 +13,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; import com.google.protobuf.UnsafeByteOperations; import org.slf4j.Logger; @@ -21,10 +20,7 @@ import tech.ydb.common.transaction.YdbTransaction; import tech.ydb.core.Status; -import tech.ydb.core.metrics.Meter; import tech.ydb.topic.description.Codec; -import tech.ydb.topic.description.CodecRegistry; -import tech.ydb.topic.settings.WriterSettings; import tech.ydb.topic.write.Message; import tech.ydb.topic.write.QueueOverflowException; import tech.ydb.topic.write.WriteAck; @@ -33,7 +29,7 @@ * * @author Aleksandr Gorshenin */ -public class WriterQueue { +class WriterQueue { interface EncodedMsg { SentMessage getSentMessage(); @@ -61,25 +57,13 @@ interface EncodedMsg { // Future for flush method. Only the future is stored to not retain the message data after sending private volatile CompletableFuture lastAcceptedAckFuture = null; - @VisibleForTesting - WriterQueue(String debugId, WriterSettings settings, CodecRegistry codecRegistry, - Executor compressionExecutor, Runnable readyNotify) { - this(debugId, settings, new WriterMetrics(Meter.NOOP, settings.getTopicPath(), settings.getWriterName()), - codecRegistry, compressionExecutor, readyNotify); - } - - public WriterQueue(String debugId, WriterSettings settings, WriterMetrics metrics, CodecRegistry codecRegistry, - Executor compressionExecutor, Runnable readyNotify) { + WriterQueue(String debugId, WriteConfig config, Runnable readyNotify) { this.debugId = debugId; - this.buffer = new BufferManager(debugId, settings); - - this.codec = codecRegistry.getCodec(settings.getCodec()); - if (codec == null) { - throw new IllegalArgumentException("Unsupported codec: " + settings.getCodec()); - } - this.compressionExecutor = compressionExecutor; + this.buffer = new BufferManager(debugId, config.getBufferMaxSize(), config.getBufferMaxMessagesCount()); + this.codec = config.getCodec(); + this.compressionExecutor = config.getCompressor(); this.readyNotify = readyNotify; - this.metrics = metrics; + this.metrics = config.getMetrics(); this.metrics.register(buffer::getUsedSize, buffer::getMaxSize, this::getOlderAcceptedTimestamp); } diff --git a/topic/src/test/java/tech/ydb/topic/write/impl/BufferManagerTest.java b/topic/src/test/java/tech/ydb/topic/write/impl/BufferManagerTest.java index 4ba5488b8..7d7b22488 100644 --- a/topic/src/test/java/tech/ydb/topic/write/impl/BufferManagerTest.java +++ b/topic/src/test/java/tech/ydb/topic/write/impl/BufferManagerTest.java @@ -13,7 +13,6 @@ import tech.ydb.core.Status; import tech.ydb.core.StatusCode; -import tech.ydb.topic.settings.WriterSettings; import tech.ydb.topic.write.QueueOverflowException; /** @@ -21,15 +20,6 @@ */ public class BufferManagerTest { - private static BufferManager manager(long maxBytes, int maxCount) { - return new BufferManager("test", WriterSettings.newBuilder() - .setTopicPath("/test") - .setMaxSendBufferMemorySize(maxBytes) - .setMaxSendBufferMessagesCount(maxCount) - .build() - ); - } - private static void assertOverflow(String msg, ThrowingRunnable runnable) { QueueOverflowException ex = Assert.assertThrows("Must be thrown QueueOverflowException", QueueOverflowException.class, runnable); @@ -75,13 +65,13 @@ private static void assertInterrupted(ThrowingRunnable runnable) throws Interrup @Test public void testGetMaxSize() { - BufferManager bm = manager(1024, 10); + BufferManager bm = new BufferManager("test", 1024, 10); Assert.assertEquals(1024, bm.getMaxSize()); } @Test public void testAcquireAndRelease() throws Exception { - BufferManager bm = manager(70, 5); + BufferManager bm = new BufferManager("test", 70, 5); bm.acquire(30); // success bm.acquire(30); // success assertOverflow("[test] Rejecting a message of 40 bytes: not enough space in message queue. " @@ -108,7 +98,7 @@ public void testAcquireAndRelease() throws Exception { @Test public void testAcquireReleasesCountOnByteInterrupt() throws Exception { - BufferManager bm = manager(20, 2); + BufferManager bm = new BufferManager("test", 20, 2); // Fill bytes completely: acquire 10 bytes bm.acquire(10); @@ -125,14 +115,14 @@ public void testAcquireReleasesCountOnByteInterrupt() throws Exception { @Test public void testTryAcquireWithTimeoutSuccess() throws Exception { - BufferManager bm = manager(100, 5); + BufferManager bm = new BufferManager("test", 100, 5); bm.tryAcquire(50, 100, TimeUnit.MILLISECONDS); bm.releaseMessage(50); } @Test public void testTryAcquireWithTimeoutBytesTimeoutThrows() throws Exception { - BufferManager bm = manager(20, 2); + BufferManager bm = new BufferManager("test", 20, 2); bm.tryAcquire(15, 1, TimeUnit.MILLISECONDS); // success assertTimeout("[test] Rejecting a message of 15 bytes: not enough space in message queue. " @@ -160,7 +150,7 @@ public void testTryAcquireWithTimeoutBytesTimeoutThrows() throws Exception { @Test public void testUpdateMessageSizeAcquiresAvailableCapacity() throws Exception { - BufferManager bufferManager = manager(10, 3); + BufferManager bufferManager = new BufferManager("test", 10, 3); bufferManager.tryAcquire(4); bufferManager.tryAcquire(4); @@ -180,14 +170,14 @@ public void testUpdateMessageSizeAcquiresAvailableCapacity() throws Exception { @Test public void testLargeBufferConfiguration() throws QueueOverflowException { - BufferManager bm = manager(Integer.MAX_VALUE, 10); + BufferManager bm = new BufferManager("test", Integer.MAX_VALUE, 10); bm.tryAcquire(Integer.MAX_VALUE); //success assertOverflow("[test] Rejecting a message of 1 bytes: not enough space in message queue. " + "Buffer currently has 1 messages with 0 / 2147483647 bytes available", () -> bm.tryAcquire(1)); bm.releaseMessage(Integer.MAX_VALUE); // success - BufferManager bm2 = manager(0x80000000L, 10); // MAX_VALUE + 1 = 2GB + BufferManager bm2 = new BufferManager("test", 0x80000000L, 10); // MAX_VALUE + 1 = 2GB bm2.tryAcquire(0x40000000); bm2.tryAcquire(0x40000000); assertOverflow("[test] Rejecting a message of 1 bytes: not enough space in message queue. " @@ -196,7 +186,7 @@ public void testLargeBufferConfiguration() throws QueueOverflowException { bm2.releaseMessage(0x40000000); bm2.releaseMessage(0x40000000); - BufferManager bm3 = manager(0x800000000L, 100); // 32 GB + BufferManager bm3 = new BufferManager("test", 0x800000000L, 100); // 32 GB for (int idx = 0; idx < 64; idx++) { bm3.tryAcquire(0x20000000); } @@ -208,7 +198,7 @@ public void testLargeBufferConfiguration() throws QueueOverflowException { } IllegalArgumentException ex = Assert.assertThrows(IllegalArgumentException.class, - () -> manager(0x20000000000L, 1000000)); // 2024 GB + () -> new BufferManager("test", 0x20000000000L, 1000000)); // 2024 GB Assert.assertEquals("Writer buffer size must be less than 1024 GB", ex.getMessage()); } @@ -216,7 +206,7 @@ public void testLargeBufferConfiguration() throws QueueOverflowException { public void testLargeBufferSize() throws Exception { // With maxBytes = 0x100000003L, calculateBlockSize returns 2 (blockBitsCount=2) // so all messages are rounded up to 4-byte blocks - BufferManager bm = manager(0x100000003L, 10); + BufferManager bm = new BufferManager("test", 0x100000003L, 10); Assert.assertEquals(0x100000003L, bm.getMaxSize()); // every message acquire 1 block = 4 bytes @@ -243,7 +233,7 @@ public void testLargeBufferSize() throws Exception { @Test public void testClosedBuffer() { - BufferManager bm = manager(100, 3); + BufferManager bm = new BufferManager("test", 100, 3); bm.close(Status.SUCCESS); @@ -257,7 +247,7 @@ public void testClosedBuffer() { @Test public void testReleaseCountOnBufferClosing() throws InterruptedException, QueueOverflowException { - BufferManager bm = manager(100, 3); + BufferManager bm = new BufferManager("test", 100, 3); bm.acquire(10); bm.acquire(10); @@ -305,7 +295,7 @@ public void testReleaseCountOnBufferClosing() throws InterruptedException, Queue @Test public void testReleaseSizeOnBufferClosing() throws InterruptedException, QueueOverflowException { - BufferManager bm = manager(70, 5); + BufferManager bm = new BufferManager("test", 70, 5); bm.acquire(20); bm.acquire(20); diff --git a/topic/src/test/java/tech/ydb/topic/write/impl/WriterQueueTest.java b/topic/src/test/java/tech/ydb/topic/write/impl/WriterQueueTest.java index 8a067ba71..5b412e2b9 100644 --- a/topic/src/test/java/tech/ydb/topic/write/impl/WriterQueueTest.java +++ b/topic/src/test/java/tech/ydb/topic/write/impl/WriterQueueTest.java @@ -35,6 +35,8 @@ */ public class WriterQueueTest { private static final Message SMALL_MSG = Message.of(new byte[]{0x00, 0x01, 0x02, 0x03, 0x05}); + private static final CodecRegistry REGISTRY = new CodecRegistry(); + private static final Runnable NOPE = () -> { }; @Rule public final HideLoggersRule hideLogger = new HideLoggersRule(); @@ -57,9 +59,8 @@ private static WriterSettings gzipSettings() { .build(); } - private static WriterQueue rawQueue(AtomicInteger notifyCount) { - return new WriterQueue("test", rawSettings(), new CodecRegistry(), - Runnable::run, () -> notifyCount.incrementAndGet()); + private static WriteConfig rawConfig() { + return new WriteConfig(REGISTRY, Runnable::run, rawSettings()); } private static void assertOverflow(String msg, ThrowingRunnable runnable) { @@ -88,13 +89,13 @@ public void testInvalidCodecThrows() { .setCodec(9999) .build(); IllegalArgumentException ex = Assert.assertThrows(IllegalArgumentException.class, - () -> new WriterQueue("test", settings, new CodecRegistry(), Runnable::run, () -> { })); + () -> new WriteConfig(REGISTRY, Runnable::run, settings)); Assert.assertEquals("Unsupported codec: 9999", ex.getMessage()); } @Test public void testFlushWithNoMessages() { - WriterQueue q = rawQueue(new AtomicInteger()); + WriterQueue q = new WriterQueue("fwnn", rawConfig(), NOPE); CompletableFuture f = q.flush(); Assert.assertTrue(f.isDone()); Assert.assertFalse(f.isCompletedExceptionally()); @@ -102,7 +103,7 @@ public void testFlushWithNoMessages() { @Test public void testRawCompressor() throws Exception { - WriterQueue q = new WriterQueue("test", rawSettings(), new CodecRegistry(), null, () -> {}); + WriterQueue q = new WriterQueue("rc", rawConfig(), NOPE); CompletableFuture f1 = q.enqueue(SMALL_MSG, null); CompletableFuture f2 = q.tryEnqueue(SMALL_MSG, null); @@ -127,19 +128,18 @@ public void testRawCompressor() throws Exception { @Test public void testCompressedExpansionIsAccounted() throws Exception { - Codec codec = expandingCodec(Codec.GZIP, 1); + Codec codec = new ExplandingCodec(Codec.GZIP, 1); + CodecRegistry registry = new CodecRegistry(singletonList(codec)); WriterSettings settings = WriterSettings.newBuilder() .setTopicPath("/test") .setCodec(codec.getId()) .setMaxSendBufferMemorySize(10) .build(); - WriterQueue writerQueue = new WriterQueue("test", settings, new CodecRegistry(singletonList(codec)), - Runnable::run, () -> { - }); + WriterQueue writerQueue = new WriterQueue("cea", new WriteConfig(registry, Runnable::run, settings), NOPE); writerQueue.tryEnqueue(SMALL_MSG, null); assertOverflow( - "[test] Rejecting a message of 5 bytes: not enough space in message queue. " + "[cea] Rejecting a message of 5 bytes: not enough space in message queue. " + "Buffer currently has 1 messages with 4 / 10 bytes available", () -> writerQueue.tryEnqueue(SMALL_MSG, null) ); @@ -153,14 +153,13 @@ public void testLzopFramingIsAccounted() throws Exception { .setCodec(Codec.LZOP) .setMaxSendBufferMemorySize(109) .build(); - WriterQueue writerQueue = new WriterQueue("test", settings, new CodecRegistry(), encodingTasks::add, () -> { - }); + WriterQueue queue = new WriterQueue("lfa", new WriteConfig(REGISTRY, encodingTasks::add, settings), NOPE); - writerQueue.tryEnqueue(SMALL_MSG, null); + queue.tryEnqueue(SMALL_MSG, null); assertOverflow( - "[test] Rejecting a message of 55 bytes: not enough space in message queue. " + "[lfa] Rejecting a message of 55 bytes: not enough space in message queue. " + "Buffer currently has 1 messages with 54 / 109 bytes available", - () -> writerQueue.tryEnqueue(SMALL_MSG, null) + () -> queue.tryEnqueue(SMALL_MSG, null) ); } @@ -174,81 +173,79 @@ public void testBuiltInCodecSizeBoundsDoNotOverflow() { } @Test - @HideLoggers({WriterImpl.class}) + @HideLoggers(WriterImpl.class) public void testEncodedMessageLargerThanBufferIsProcessed() throws Exception { - Codec codec = expandingCodec(9004, 6); + Codec codec = new ExplandingCodec(9004, 6); + CodecRegistry registry = new CodecRegistry(singletonList(codec)); Queue encodingTasks = new ArrayDeque<>(); WriterSettings settings = WriterSettings.newBuilder() .setTopicPath("/test") .setCodec(codec.getId()) .setMaxSendBufferMemorySize(10) .build(); - WriterQueue writerQueue = new WriterQueue("test", settings, new CodecRegistry(singletonList(codec)), - encodingTasks::add, () -> { - }); - writerQueue.tryEnqueue(SMALL_MSG, null); - writerQueue.tryEnqueue(SMALL_MSG, null); + WriterQueue queue = new WriterQueue("emltbp", new WriteConfig(registry, encodingTasks::add, settings), NOPE); + queue.tryEnqueue(SMALL_MSG, null); + queue.tryEnqueue(SMALL_MSG, null); encodingTasks.remove().run(); - SentMessage first = writerQueue.nextMessageToSend(); + SentMessage first = queue.nextMessageToSend(); Assert.assertNotNull(first); Assert.assertEquals(11, first.getPb().getData().size()); Assert.assertEquals(5, first.getBufferSize()); - writerQueue.confirmAck(new WriteAck(first.getSeqNo(), WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(first.getSeqNo(), WriteAck.State.WRITTEN, null, null)); encodingTasks.remove().run(); - SentMessage second = writerQueue.nextMessageToSend(); + SentMessage second = queue.nextMessageToSend(); Assert.assertNotNull(second); Assert.assertEquals(11, second.getPb().getData().size()); Assert.assertEquals(10, second.getBufferSize()); } @Test - @HideLoggers({WriterImpl.class}) + @HideLoggers(WriterImpl.class) public void testExpansionUsesAvailableCapacity() throws Exception { - Codec codec = expandingCodec(9005, 4); + Codec codec = new ExplandingCodec(9005, 4); + CodecRegistry registry = new CodecRegistry(singletonList(codec)); Queue encodingTasks = new ArrayDeque<>(); WriterSettings settings = WriterSettings.newBuilder() .setTopicPath("/test") .setCodec(codec.getId()) .setMaxSendBufferMemorySize(12) .build(); - WriterQueue writerQueue = new WriterQueue("test", settings, new CodecRegistry(singletonList(codec)), - encodingTasks::add, () -> { - }); + WriterQueue queue = new WriterQueue("euac", new WriteConfig(registry, encodingTasks::add, settings), NOPE); - writerQueue.tryEnqueue(SMALL_MSG, null); - writerQueue.tryEnqueue(SMALL_MSG, null); + queue.tryEnqueue(SMALL_MSG, null); + queue.tryEnqueue(SMALL_MSG, null); encodingTasks.remove().run(); - SentMessage first = writerQueue.nextMessageToSend(); + SentMessage first = queue.nextMessageToSend(); Assert.assertNotNull(first); Assert.assertEquals(9, first.getPb().getData().size()); Assert.assertEquals(7, first.getBufferSize()); - writerQueue.confirmAck(new WriteAck(first.getSeqNo(), WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(first.getSeqNo(), WriteAck.State.WRITTEN, null, null)); encodingTasks.remove().run(); - SentMessage second = writerQueue.nextMessageToSend(); + SentMessage second = queue.nextMessageToSend(); Assert.assertNotNull(second); Assert.assertEquals(9, second.getPb().getData().size()); Assert.assertEquals(9, second.getBufferSize()); } @Test - @HideLoggers({ WriterImpl.class }) + @HideLoggers(WriterImpl.class) public void testGzipNullCompressor() throws Exception { - WriterQueue q = new WriterQueue("test", gzipSettings(), new CodecRegistry(), null, () -> {}); + WriterQueue queue = new WriterQueue("gnp", new WriteConfig(new CodecRegistry(), null, gzipSettings()), NOPE); - CompletableFuture f1 = q.enqueue(SMALL_MSG, null); - CompletableFuture f2 = q.tryEnqueue(SMALL_MSG, null); - CompletableFuture f3 = q.tryEnqueue(SMALL_MSG, null, 1, TimeUnit.SECONDS); + CompletableFuture f1 = queue.enqueue(SMALL_MSG, null); + CompletableFuture f2 = queue.tryEnqueue(SMALL_MSG, null); + CompletableFuture f3 = queue.tryEnqueue(SMALL_MSG, null, 1, TimeUnit.SECONDS); Assert.assertFalse(f1.isDone()); Assert.assertFalse(f2.isDone()); Assert.assertFalse(f3.isDone()); - Assert.assertNull(q.nextMessageToSend()); // nothing to send, all messages were failed + Assert.assertNull(queue.nextMessageToSend()); // nothing to send, all messages were failed Assert.assertTrue(f1.isCompletedExceptionally()); Assert.assertTrue(f2.isCompletedExceptionally()); @@ -256,7 +253,7 @@ public void testGzipNullCompressor() throws Exception { } @Test - @HideLoggers({WriterImpl.class}) + @HideLoggers(WriterImpl.class) public void testWrongCodec() throws Exception { // Codec that always throws on encode Codec failingCodec = new Codec() { @@ -276,24 +273,25 @@ public OutputStream encode(OutputStream out) throws IOException { } }; - AtomicInteger notify = new AtomicInteger(); CodecRegistry registry = new CodecRegistry(Arrays.asList(failingCodec)); WriterSettings settings = WriterSettings.newBuilder() .setTopicPath("/test") .setCodec(failingCodec.getId()) .build(); + WriteConfig config = new WriteConfig(registry, Runnable::run, settings); - WriterQueue q = new WriterQueue("test", settings, registry, Runnable::run, notify::incrementAndGet); + AtomicInteger notify = new AtomicInteger(); + WriterQueue queue = new WriterQueue("wc", config, notify::incrementAndGet); - CompletableFuture f1 = q.enqueue(SMALL_MSG, null); - CompletableFuture f2 = q.tryEnqueue(SMALL_MSG, null); - CompletableFuture f3 = q.tryEnqueue(SMALL_MSG, null, 1, TimeUnit.SECONDS); + CompletableFuture f1 = queue.enqueue(SMALL_MSG, null); + CompletableFuture f2 = queue.tryEnqueue(SMALL_MSG, null); + CompletableFuture f3 = queue.tryEnqueue(SMALL_MSG, null, 1, TimeUnit.SECONDS); Assert.assertFalse(f1.isDone()); Assert.assertFalse(f2.isDone()); Assert.assertFalse(f3.isDone()); - Assert.assertNull(q.nextMessageToSend()); // nothing to send, all messages were failed + Assert.assertNull(queue.nextMessageToSend()); // nothing to send, all messages were failed Assert.assertTrue(f1.isCompletedExceptionally()); Assert.assertTrue(f2.isCompletedExceptionally()); @@ -301,7 +299,7 @@ public OutputStream encode(OutputStream out) throws IOException { } @Test - @HideLoggers({ WriterImpl.class }) + @HideLoggers(WriterImpl.class) public void badMsgBehindSentTest() throws Exception { // codec that fails only for messages whose first byte is 0x7F Codec selective = new Codec() { @@ -333,26 +331,28 @@ public void write(int b) throws IOException { .setTopicPath("/test") .setCodec(selective.getId()) .build(); - AtomicInteger notify = new AtomicInteger(); CodecRegistry registry = new CodecRegistry(Arrays.asList(selective)); - WriterQueue q = new WriterQueue("test", settings, registry, Runnable::run, notify::incrementAndGet); + WriteConfig config = new WriteConfig(registry, Runnable::run, settings); - CompletableFuture good1 = q.enqueue(Message.of(new byte[] { 0x01 }), null); - CompletableFuture bad = q.enqueue(Message.of(new byte[] { 0x7F }), null); - CompletableFuture good2 = q.enqueue(Message.of(new byte[] { 0x02 }), null); + AtomicInteger notify = new AtomicInteger(); + WriterQueue queue = new WriterQueue("bmbst", config, notify::incrementAndGet); + + CompletableFuture good1 = queue.enqueue(Message.of(new byte[] { 0x01 }), null); + CompletableFuture bad = queue.enqueue(Message.of(new byte[] { 0x7F }), null); + CompletableFuture good2 = queue.enqueue(Message.of(new byte[] { 0x02 }), null); - SentMessage s1 = q.nextMessageToSend(); + SentMessage s1 = queue.nextMessageToSend(); Assert.assertNotNull(s1); - SentMessage s2 = q.nextMessageToSend(); + SentMessage s2 = queue.nextMessageToSend(); Assert.assertNotNull(s2); - Assert.assertNull(q.nextMessageToSend()); + Assert.assertNull(queue.nextMessageToSend()); Assert.assertFalse(good1.isDone()); Assert.assertFalse(good2.isDone()); Assert.assertFalse(bad.isDone()); // ack for the first message only -> sweep reaches the ProblemMsg - q.confirmAck(new WriteAck(s1.getSeqNo(), WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(s1.getSeqNo(), WriteAck.State.WRITTEN, null, null)); Assert.assertTrue(good1.isDone()); Assert.assertTrue(bad.isCompletedExceptionally()); @@ -363,18 +363,18 @@ public void write(int b) throws IOException { @Test public void testFlushCompletesWhenMessageIsAcked() throws Exception { AtomicInteger notify = new AtomicInteger(); - WriterQueue q = rawQueue(notify); + WriterQueue queue = new WriterQueue("fcwma", rawConfig(), notify::incrementAndGet); - q.enqueue(SMALL_MSG, null); - q.enqueue(SMALL_MSG, null); - q.enqueue(SMALL_MSG, null); + queue.enqueue(SMALL_MSG, null); + queue.enqueue(SMALL_MSG, null); + queue.enqueue(SMALL_MSG, null); - CompletableFuture flushFuture = q.flush(); + CompletableFuture flushFuture = queue.flush(); - long lastSeqNo = assertSendAll(q, 3); + long lastSeqNo = assertSendAll(queue, 3); Assert.assertFalse(flushFuture.isDone()); - q.confirmAck(new WriteAck(lastSeqNo, WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(lastSeqNo, WriteAck.State.WRITTEN, null, null)); Assert.assertTrue(flushFuture.isDone()); Assert.assertFalse(flushFuture.isCompletedExceptionally()); @@ -382,15 +382,15 @@ public void testFlushCompletesWhenMessageIsAcked() throws Exception { @Test public void testFlushCompletesWhenMessageIsFailed() throws Exception { - WriterQueue q = rawQueue(new AtomicInteger()); + WriterQueue queue = new WriterQueue("fcwmf", rawConfig(), NOPE); - CompletableFuture ackFuture = q.enqueue(SMALL_MSG, null); - assertSendAll(q, 1); + CompletableFuture ackFuture = queue.enqueue(SMALL_MSG, null); + assertSendAll(queue, 1); - CompletableFuture flushFuture = q.flush(); + CompletableFuture flushFuture = queue.flush(); Assert.assertFalse(flushFuture.isDone()); - q.close(Status.of(StatusCode.CLIENT_CANCELLED)); + queue.close(Status.of(StatusCode.CLIENT_CANCELLED)); Assert.assertTrue(ackFuture.isCompletedExceptionally()); Assert.assertTrue(flushFuture.isDone()); @@ -399,43 +399,44 @@ public void testFlushCompletesWhenMessageIsFailed() throws Exception { @Test public void testSmallBufferWriting() throws QueueOverflowException { - AtomicInteger notify = new AtomicInteger(); WriterSettings settings = WriterSettings.newBuilder() .setTopicPath("/test") .setCodec(Codec.RAW) .setMaxSendBufferMemorySize(12) .build(); + WriteConfig config = new WriteConfig(REGISTRY, null, settings); - WriterQueue q = new WriterQueue("test", settings, new CodecRegistry(), null, notify::incrementAndGet); + AtomicInteger notify = new AtomicInteger(); + WriterQueue queue = new WriterQueue("sbw", config, notify::incrementAndGet); - q.tryEnqueue(smallMsg(10), null); // success - q.tryEnqueue(smallMsg(20), null); // success - assertOverflow("[test] Rejecting a message of 5 bytes: not enough space in message queue. " + queue.tryEnqueue(smallMsg(10), null); // success + queue.tryEnqueue(smallMsg(20), null); // success + assertOverflow("[sbw] Rejecting a message of 5 bytes: not enough space in message queue. " + "Buffer currently has 2 messages with 2 / 12 bytes available", - () -> q.tryEnqueue(smallMsg(30), null)); + () -> queue.tryEnqueue(smallMsg(30), null)); - Assert.assertEquals(20, assertSendAll(q, 2)); - q.confirmAck(new WriteAck(10, null, null, null)); // free one message + Assert.assertEquals(20, assertSendAll(queue, 2)); + queue.confirmAck(new WriteAck(10, null, null, null)); // free one message - q.tryEnqueue(smallMsg(30), null); // success - assertOverflow("[test] Rejecting a message of 5 bytes: not enough space in message queue. " + queue.tryEnqueue(smallMsg(30), null); // success + assertOverflow("[sbw] Rejecting a message of 5 bytes: not enough space in message queue. " + "Buffer currently has 2 messages with 2 / 12 bytes available", - () -> q.tryEnqueue(smallMsg(40), null)); + () -> queue.tryEnqueue(smallMsg(40), null)); - Assert.assertEquals(30, assertSendAll(q, 1)); - q.confirmAck(new WriteAck(30, null, null, null)); // free one message + Assert.assertEquals(30, assertSendAll(queue, 1)); + queue.confirmAck(new WriteAck(30, null, null, null)); // free one message } @Test public void testIncorrectSeqNumbers() throws Exception { - WriterQueue q = rawQueue(new AtomicInteger()); + WriterQueue queue = new WriterQueue("isn", rawConfig(), NOPE); - CompletableFuture f1 = q.enqueue(smallMsg(10), null); - CompletableFuture f2 = q.enqueue(smallMsg(20), null); - CompletableFuture f3 = q.enqueue(smallMsg(20), null); - CompletableFuture f4 = q.enqueue(smallMsg(30), null); - CompletableFuture f5 = q.enqueue(smallMsg(11), null); - CompletableFuture f6 = q.enqueue(smallMsg(40), null); + CompletableFuture f1 = queue.enqueue(smallMsg(10), null); + CompletableFuture f2 = queue.enqueue(smallMsg(20), null); + CompletableFuture f3 = queue.enqueue(smallMsg(20), null); + CompletableFuture f4 = queue.enqueue(smallMsg(30), null); + CompletableFuture f5 = queue.enqueue(smallMsg(11), null); + CompletableFuture f6 = queue.enqueue(smallMsg(40), null); Assert.assertFalse(f1.isDone()); Assert.assertFalse(f2.isDone()); @@ -444,10 +445,10 @@ public void testIncorrectSeqNumbers() throws Exception { Assert.assertFalse(f5.isDone()); Assert.assertFalse(f6.isDone()); - long lastSeqNo = assertSendAll(q, 6); + long lastSeqNo = assertSendAll(queue, 6); Assert.assertEquals(40, lastSeqNo); // all message must be resent - List resend = q.updateSeqNo(15); + List resend = queue.updateSeqNo(15); Assert.assertEquals(6, resend.size()); Assert.assertEquals(10, resend.get(0).getSeqNo()); @@ -465,12 +466,12 @@ public void testIncorrectSeqNumbers() throws Exception { Assert.assertFalse(f6.isDone()); // all message must be confirmed - q.confirmAck(new WriteAck(10, WriteAck.State.ALREADY_WRITTEN, null, null)); - q.confirmAck(new WriteAck(20, WriteAck.State.WRITTEN, null, null)); - q.confirmAck(new WriteAck(20, WriteAck.State.ALREADY_WRITTEN, null, null)); // will be skipped - q.confirmAck(new WriteAck(30, WriteAck.State.WRITTEN, null, null)); - q.confirmAck(new WriteAck(11, WriteAck.State.ALREADY_WRITTEN, null, null)); // will be skipped - q.confirmAck(new WriteAck(40, WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(10, WriteAck.State.ALREADY_WRITTEN, null, null)); + queue.confirmAck(new WriteAck(20, WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(20, WriteAck.State.ALREADY_WRITTEN, null, null)); // will be skipped + queue.confirmAck(new WriteAck(30, WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(11, WriteAck.State.ALREADY_WRITTEN, null, null)); // will be skipped + queue.confirmAck(new WriteAck(40, WriteAck.State.WRITTEN, null, null)); Assert.assertTrue(f1.isDone()); Assert.assertTrue(f2.isDone()); @@ -495,16 +496,16 @@ public void testIncorrectSeqNumbers() throws Exception { @Test public void testLostAcks() throws Exception { - WriterQueue q = rawQueue(new AtomicInteger()); + WriterQueue queue = new WriterQueue("la", rawConfig(), NOPE); - List before = q.updateSeqNo(0); + List before = queue.updateSeqNo(0); Assert.assertTrue(before.isEmpty()); - CompletableFuture f1 = q.enqueue(smallMsg(10), null); - CompletableFuture f2 = q.enqueue(smallMsg(20), null); - CompletableFuture f3 = q.enqueue(smallMsg(30), null); - CompletableFuture f4 = q.enqueue(smallMsg(40), null); - CompletableFuture f5 = q.enqueue(smallMsg(50), null); + CompletableFuture f1 = queue.enqueue(smallMsg(10), null); + CompletableFuture f2 = queue.enqueue(smallMsg(20), null); + CompletableFuture f3 = queue.enqueue(smallMsg(30), null); + CompletableFuture f4 = queue.enqueue(smallMsg(40), null); + CompletableFuture f5 = queue.enqueue(smallMsg(50), null); Assert.assertFalse(f1.isDone()); Assert.assertFalse(f2.isDone()); @@ -512,10 +513,10 @@ public void testLostAcks() throws Exception { Assert.assertFalse(f4.isDone()); Assert.assertFalse(f5.isDone()); - long lastSeqNo = assertSendAll(q, 5); + long lastSeqNo = assertSendAll(queue, 5); Assert.assertEquals(50, lastSeqNo); - q.confirmAck(new WriteAck(10, WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(10, WriteAck.State.WRITTEN, null, null)); Assert.assertTrue(f1.isDone()); Assert.assertEquals(WriteAck.State.WRITTEN, f1.join().getState()); @@ -525,7 +526,7 @@ public void testLostAcks() throws Exception { Assert.assertFalse(f5.isDone()); // lost others acks and reconnect with new lastSeqNo - List retry = q.updateSeqNo(30); + List retry = queue.updateSeqNo(30); // All messages without acks will be resent Assert.assertEquals(4, retry.size()); @@ -539,10 +540,10 @@ public void testLostAcks() throws Exception { Assert.assertFalse(f4.isDone()); Assert.assertFalse(f5.isDone()); - q.confirmAck(new WriteAck(20, WriteAck.State.ALREADY_WRITTEN, null, null)); - q.confirmAck(new WriteAck(30, WriteAck.State.ALREADY_WRITTEN, null, null)); - q.confirmAck(new WriteAck(40, WriteAck.State.WRITTEN, null, null)); - q.confirmAck(new WriteAck(50, WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(20, WriteAck.State.ALREADY_WRITTEN, null, null)); + queue.confirmAck(new WriteAck(30, WriteAck.State.ALREADY_WRITTEN, null, null)); + queue.confirmAck(new WriteAck(40, WriteAck.State.WRITTEN, null, null)); + queue.confirmAck(new WriteAck(50, WriteAck.State.WRITTEN, null, null)); Assert.assertTrue(f2.isDone()); Assert.assertTrue(f3.isDone()); @@ -554,28 +555,34 @@ public void testLostAcks() throws Exception { Assert.assertEquals(WriteAck.State.WRITTEN, f5.join().getState()); } - private static Codec expandingCodec(int id, int additionalBytes) { - return new Codec() { - @Override - public int getId() { - return id; - } + private class ExplandingCodec implements Codec { + private final int id; + private final int prefixSize; - @Override - public InputStream decode(InputStream inputStream) { - return inputStream; - } + public ExplandingCodec(int id, int additionalBytes) { + this.id = id; + this.prefixSize = additionalBytes; + } - @Override - public OutputStream encode(OutputStream outputStream) { - return new FilterOutputStream(outputStream) { - @Override - public void close() throws IOException { - out.write(new byte[additionalBytes]); - super.close(); - } - }; - } - }; + @Override + public int getId() { + return id; + } + + @Override + public InputStream decode(InputStream inputStream) { + return inputStream; + } + + @Override + public OutputStream encode(OutputStream outputStream) { + return new FilterOutputStream(outputStream) { + @Override + public void close() throws IOException { + out.write(new byte[prefixSize]); + super.close(); + } + }; + } } } From 791cd7964e58e795bd670d747f8249b8af36351f Mon Sep 17 00:00:00 2001 From: Alexandr Gorshenin Date: Tue, 27 Oct 2026 12:16:54 +0000 Subject: [PATCH 2/2] Moved metrics from ReadPartitionDecoder to ReadPartition --- .../ydb/topic/read/impl/MessageDecoder.java | 14 +++++++---- .../ydb/topic/read/impl/ReadPartition.java | 19 ++++++++------- .../topic/read/impl/ReadPartitionDecoder.java | 10 +------- .../tech/ydb/topic/read/impl/ReadSession.java | 20 +++++++++------- .../topic/read/impl/MessageDecoderTest.java | 24 ++++++++++--------- 5 files changed, 44 insertions(+), 43 deletions(-) diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/MessageDecoder.java b/topic/src/main/java/tech/ydb/topic/read/impl/MessageDecoder.java index b52b26cd1..6f1c35077 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/MessageDecoder.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/MessageDecoder.java @@ -21,25 +21,29 @@ public class MessageDecoder { private final long maxBufferSize; private final AtomicLong totalAvailable; - private final Executor decompressionExecutor; + private final Executor decompressor; private final CodecRegistry codecRegistry; + private final ReaderMetrics metrics; + private final Queue decodingQueue = new ConcurrentLinkedQueue<>(); private final SerialRunnable decodeNext = new SerialRunnable(new DecodeNext()); private volatile boolean isStopped = false; public MessageDecoder(ReadConfig config) { - this(config.getMaxMemoryUsageBytes(), config.getDecompressor(), config.getCodecRegistry()); + this(config.getMaxMemoryUsageBytes(), config.getDecompressor(), config.getCodecRegistry(), config.getMetrics()); } - MessageDecoder(long maxBufferSize, Executor decompressionExecutor, CodecRegistry codecRegistry) { + @VisibleForTesting + MessageDecoder(long maxBufferSize, Executor decompressor, CodecRegistry codecRegistry, ReaderMetrics metrics) { if (maxBufferSize <= 0) { throw new IllegalArgumentException("maxBufferSize must be positive, but got " + maxBufferSize); } this.maxBufferSize = maxBufferSize; this.totalAvailable = new AtomicLong(maxBufferSize); - this.decompressionExecutor = decompressionExecutor; + this.decompressor = decompressor; this.codecRegistry = codecRegistry; + this.metrics = metrics; } public void decodeNext() { @@ -96,7 +100,7 @@ public void run() { long size = next.allocate(); totalAvailable.addAndGet(-size); try { - decompressionExecutor.execute(() -> next.decode(codecRegistry)); + decompressor.execute(() -> next.decode(codecRegistry, metrics)); } catch (Throwable ex) { logger.error("Cannot execute decompression ", ex); next.setError(ex); diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java index 95b36118d..ec4c7c8d3 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartition.java @@ -53,6 +53,7 @@ private enum State { private final ReaderMetrics metrics; private final Executor dataExecutor; private final Executor controlExecutor; + private final ReadSession.Handler handler; private final ReadPartitionCommitter committer; private final ReadPartitionDecoder decoder; @@ -68,15 +69,15 @@ private enum State { this.session = session; this.partition = partition; + this.handler = session.getHandler(); ReadConfig config = session.getConfig(); - this.metrics = config.getMetrics(); this.dataExecutor = new SerialExecutor(config.getDataExecutor()); this.controlExecutor = config.getControlExecutor(); this.committer = new ReadPartitionCommitter(traceID, session, this, lastCommittedOffset); this.decoder = new ReadPartitionDecoder(traceID, session.getDecoder(), partition, committer, - this::sendDataToReaders, metrics); + this::sendDataToReaders); this.queue = new ReadPartitionQueue(traceID, decoder, config.getMaxBatchSize(), lastCommittedOffset); logger.info("[{}] started for {}", traceID, partition); @@ -150,7 +151,7 @@ public boolean addBatches(List ba private void sendCommitOffsetAck() { CommitOffsetAcknowledgementEvent event = commitOffsetAck.getAndSet(null); if (event != null) { - session.getHandler().onCommitAck(event); + handler.onCommitAck(event); } } @@ -174,7 +175,7 @@ private void sendDataToReaders() { long offsetEnd = event.getMessages().get(event.getMessages().size() - 1).getOffset(); logger.debug("[{}] onData with {} message(s) (offsets {}-{}) is about to be called...", traceID, messagesCount, offsetStart, offsetEnd); - session.getHandler().onData(event); + handler.onData(event); logger.debug("[{}] onData with {} message(s) (offsets {}-{}) successfully finished", traceID, messagesCount, offsetStart, offsetEnd); @@ -186,7 +187,7 @@ private void sendDataToReaders() { if (event != null) { logger.info("[{}] has finished processing and unpaused child partitions {}", traceID, event.getChildsString()); - session.getHandler().onPartitionEnded(event); + handler.onPartitionEnded(event); session.releaseLocks(partition, event.getChilds()); } } @@ -213,7 +214,7 @@ public void close() { if (old != State.STOPPED && old != State.CREATED) { PartitionSessionClosedEventImpl closedEvent = new PartitionSessionClosedEventImpl(partition); // partition close event doesn't use partition's executors - controlExecutor.execute(() -> session.getHandler().onPartitionClosed(closedEvent)); + controlExecutor.execute(() -> handler.onPartitionClosed(closedEvent)); } } @@ -227,7 +228,7 @@ public void start(long committed, OffsetsRange offsets) { // partition start event doesn't use partition's executors controlExecutor.execute(() -> { if (state.compareAndSet(State.CREATED, State.INITED)) { - session.getHandler().onPartitionStarted(new StartEvent(committed, offsets)); + handler.onPartitionStarted(new StartEvent(committed, offsets)); return; } logger.info("[{}] skipped start confirmation event because is already {}", traceID, state.get()); @@ -251,11 +252,11 @@ public void stop(long committed) { // partition close event doesn't use partition's executors PartitionSessionClosedEventImpl event = new PartitionSessionClosedEventImpl(partition); - session.getConfig().getControlExecutor().execute(() -> session.getHandler().onPartitionClosed(event)); + controlExecutor.execute(() -> handler.onPartitionClosed(event)); return; } if (state.compareAndSet(State.STARTED, State.PRE_STOPPED)) { - session.getHandler().onPartitionStopped(new StopEvent(committed)); + handler.onPartitionStopped(new StopEvent(committed)); return; } logger.warn("[{}] skipped onStop because the partition session is already {}", traceID, state.get()); diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionDecoder.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionDecoder.java index e4c331c29..f1d16bfc7 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionDecoder.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadPartitionDecoder.java @@ -12,7 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import tech.ydb.core.metrics.Meter; import tech.ydb.proto.topic.YdbTopic; import tech.ydb.topic.description.Codec; import tech.ydb.topic.description.CodecRegistry; @@ -32,7 +31,6 @@ class ReadPartitionDecoder { private final PartitionSession partition; private final MessageCommitter committer; private final MessageDecoder decoder; - private final ReaderMetrics metrics; private final Runnable readyHandler; private final Queue allocatedMessages = new ConcurrentLinkedQueue<>(); @@ -40,14 +38,8 @@ class ReadPartitionDecoder { ReadPartitionDecoder(String traceId, MessageDecoder decoder, PartitionSession partition, MessageCommitter committer, Runnable readyHandler) { - this(traceId, decoder, partition, committer, readyHandler, new ReaderMetrics(Meter.NOOP, null, null)); - } - - ReadPartitionDecoder(String traceId, MessageDecoder decoder, PartitionSession partition, - MessageCommitter committer, Runnable readyHandler, ReaderMetrics metrics) { this.traceID = traceId; this.decoder = decoder; - this.metrics = metrics; this.partition = partition; this.committer = committer; this.readyHandler = readyHandler; @@ -179,7 +171,7 @@ public long allocate() { return uncompressedSize; } - public void decode(CodecRegistry registry) { + public void decode(CodecRegistry registry, ReaderMetrics metrics) { if (isStopped) { release(); return; diff --git a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java index ef80e9db0..31a510af0 100644 --- a/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java +++ b/topic/src/main/java/tech/ydb/topic/read/impl/ReadSession.java @@ -53,6 +53,7 @@ public interface Handler { private final ReadConfig config; private final MessageDecoder decoder; private final BufferManager bufferManager; + private final ReaderMetrics metrics; private final Handler handler; private final Map partitions = new ConcurrentHashMap<>(); @@ -66,6 +67,7 @@ public ReadSession(String id, GrpcReadWriteStream stream this.config = config; this.decoder = new MessageDecoder(config); this.bufferManager = new BufferManager(id, config.getMaxMemoryUsageBytes(), new ReadRequest()); + this.metrics = config.getMetrics(); this.handler = handler; } @@ -124,7 +126,7 @@ public boolean commitOffsets(PartitionSession session, List ranges .build(); for (OffsetsRange range : rangesToCommit) { - config.getMetrics().reportCommitQueued(session.getId(), range.getEnd(), + metrics.reportCommitQueued(session.getId(), range.getEnd(), range.getEnd() - range.getStart(), session.getPath()); } send(FromClient.newBuilder().setCommitOffsetRequest(req).build()); @@ -144,7 +146,7 @@ public void sendStartPartition(PartitionSession ps, Long readFrom, Long commitTo } if (commitTo != null) { resp.setCommitOffset(commitTo); - config.getMetrics().updateCommittedOffset(ps.getId(), commitTo); + metrics.updateCommittedOffset(ps.getId(), commitTo); } send(YdbTopic.StreamReadMessage.FromClient.newBuilder().setStartPartitionSessionResponse(resp.build()).build()); @@ -164,7 +166,7 @@ public void sendStopPartition(PartitionSession ps) { ReadPartition partition = partitions.remove(ps.getId()); if (partition != null) { - config.getMetrics().removePartition(ps.getId()); + metrics.removePartition(ps.getId()); partition.close(); } } @@ -190,7 +192,7 @@ public void removeLocks(PartitionSession ps, List childs) { public void onInit(YdbTopic.StreamReadMessage.InitResponse response) { bufferManager.init(response.getSessionId()); - config.getMetrics().register(bufferManager::getCreditBalanceBytes); + metrics.register(bufferManager::getCreditBalanceBytes); } public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionRequest req) { @@ -219,7 +221,7 @@ public void onStartPartition(YdbTopic.StreamReadMessage.StartPartitionSessionReq return; } - config.getMetrics().addPartition(psid, committed, ps.getPath()); + metrics.addPartition(psid, committed, ps.getPath()); partition.start(committed, offsets); TopicPartition lockKey = new TopicPartition(ps); if (locks.computeIfPresent(lockKey, (key, lock) -> lock.bind(partition)) != null) { @@ -258,7 +260,7 @@ public void onClosePartition(long psid) { } logger.info("[{}] Received force StopPartitionSessionRequest for {} ", debugId, partition.getPartition()); - config.getMetrics().removePartition(psid); + metrics.removePartition(psid); partition.close(); } @@ -267,7 +269,7 @@ public void onStopPartition(YdbTopic.StreamReadMessage.StopPartitionSessionReque long psid = request.getPartitionSessionId(); ReadPartition partition = partitions.get(psid); if (partition != null) { - config.getMetrics().updateCommittedOffset(psid, committedOffset); + metrics.updateCommittedOffset(psid, committedOffset); partition.stop(committedOffset); return; } @@ -281,7 +283,7 @@ public void onStopPartition(YdbTopic.StreamReadMessage.StopPartitionSessionReque public void onRead(YdbTopic.StreamReadMessage.ReadResponse response) { logger.debug("[{}] Received ReadResponse of {} bytes", debugId, response.getBytesSize()); - config.getMetrics().reportReceivedBytes(response.getBytesSize()); + metrics.reportReceivedBytes(response.getBytesSize()); bufferManager.allocate(response.getBytesSize(), response.getPartitionDataList()); for (YdbTopic.StreamReadMessage.ReadResponse.PartitionData data: response.getPartitionDataList()) { @@ -308,7 +310,7 @@ public void onCommitOffset(YdbTopic.StreamReadMessage.CommitOffsetResponse respo continue; } - config.getMetrics().reportCommitAcknowledged(offset.getPartitionSessionId(), + metrics.reportCommitAcknowledged(offset.getPartitionSessionId(), offset.getCommittedOffset(), partition.getPartition().getPath()); partition.confirmCommittedOffset(offset.getCommittedOffset()); } diff --git a/topic/src/test/java/tech/ydb/topic/read/impl/MessageDecoderTest.java b/topic/src/test/java/tech/ydb/topic/read/impl/MessageDecoderTest.java index 57dd8fd3f..ac8d16a87 100644 --- a/topic/src/test/java/tech/ydb/topic/read/impl/MessageDecoderTest.java +++ b/topic/src/test/java/tech/ydb/topic/read/impl/MessageDecoderTest.java @@ -20,6 +20,7 @@ import org.junit.Test; import org.junit.function.ThrowingRunnable; +import tech.ydb.core.metrics.Meter; import tech.ydb.proto.topic.YdbTopic.StreamReadMessage.ReadResponse; import tech.ydb.topic.description.Codec; import tech.ydb.topic.description.CodecRegistry; @@ -35,6 +36,7 @@ */ public class MessageDecoderTest { private static final CodecRegistry REGISTRY = new CodecRegistry(); + private static final ReaderMetrics NOOP_METRICS = new ReaderMetrics(Meter.NOOP, "consumer", "reader-name"); private static final PartitionSession PS1 = new PartitionSession(1, 1, "/topic"); private static final PartitionSession PS2 = new PartitionSession(2, 2, "/topic"); @@ -84,18 +86,18 @@ private static BatchMeta meta(int codec) { public void nonPositiveBufferSizeTest() { // A decoder with a non-positive budget can never admit a message and silently stalls the reader Exception ex1 = Assert.assertThrows(IllegalArgumentException.class, - () -> new MessageDecoder(0, Runnable::run, REGISTRY)); + () -> new MessageDecoder(0, Runnable::run, REGISTRY, NOOP_METRICS)); Assert.assertEquals("maxBufferSize must be positive, but got 0", ex1.getMessage()); Exception ex2 = Assert.assertThrows(IllegalArgumentException.class, - () -> new MessageDecoder(-1, Runnable::run, REGISTRY)); + () -> new MessageDecoder(-1, Runnable::run, REGISTRY, NOOP_METRICS)); Assert.assertEquals("maxBufferSize must be positive, but got -1", ex2.getMessage()); } @Test @HideLoggers({ MessageDecoder.class, ReadPartition.class }) public void readyHandlerThrowsOnDecodeTest() { - MessageDecoder decoder = new MessageDecoder(1000, Runnable::run, REGISTRY); + MessageDecoder decoder = new MessageDecoder(1000, Runnable::run, REGISTRY, NOOP_METRICS); AtomicInteger ready = new AtomicInteger(0); ReadPartitionDecoder partition = new ReadPartitionDecoder("t1", decoder, PS1, null, () -> { @@ -123,7 +125,7 @@ public void readyHandlerThrowsOnErrorTest() { Executor rejecting = task -> { throw new RejectedExecutionException("executor is saturated"); }; - MessageDecoder decoder = new MessageDecoder(1000, rejecting, REGISTRY); + MessageDecoder decoder = new MessageDecoder(1000, rejecting, REGISTRY, NOOP_METRICS); AtomicInteger ready = new AtomicInteger(0); ReadPartitionDecoder partition = new ReadPartitionDecoder("t1", decoder, PS1, null, () -> { @@ -143,7 +145,7 @@ public void readyHandlerThrowsOnErrorTest() { @Test public void rawDecodeTest() { - MessageDecoder decoder = new MessageDecoder(10000, Runnable::run, REGISTRY); + MessageDecoder decoder = new MessageDecoder(10000, Runnable::run, REGISTRY, NOOP_METRICS); AtomicInteger ready = new AtomicInteger(0); ReadPartitionDecoder partition = new ReadPartitionDecoder("t1", decoder, PS1, null, ready::incrementAndGet); @@ -163,7 +165,7 @@ public void rawDecodeTest() { @Test public void flowControlByBudgetTest() { - MessageDecoder decoder = new MessageDecoder(100, Runnable::run, REGISTRY); + MessageDecoder decoder = new MessageDecoder(100, Runnable::run, REGISTRY, NOOP_METRICS); AtomicInteger ready = new AtomicInteger(0); ReadPartitionDecoder p1 = new ReadPartitionDecoder("p1", decoder, PS1, null, ready::incrementAndGet); @@ -246,7 +248,7 @@ public void flowControlByBudgetTest() { @Test public void partitionFlowTest() { - MessageDecoder decoder = new MessageDecoder(100, Runnable::run, REGISTRY); + MessageDecoder decoder = new MessageDecoder(100, Runnable::run, REGISTRY, NOOP_METRICS); AtomicInteger r1 = new AtomicInteger(0); ReadPartitionDecoder p1 = new ReadPartitionDecoder("p1", decoder, PS1, null, r1::incrementAndGet); @@ -324,7 +326,7 @@ public void partitionFlowTest() { @Test public void decodeStopTest() { - MessageDecoder decoder = new MessageDecoder(70, Runnable::run, REGISTRY); + MessageDecoder decoder = new MessageDecoder(70, Runnable::run, REGISTRY, NOOP_METRICS); AtomicInteger ready = new AtomicInteger(0); ReadPartitionDecoder partition = new ReadPartitionDecoder("t3", decoder, PS1, null, ready::incrementAndGet); @@ -365,7 +367,7 @@ public void decodeStopTest() { @Test public void decodesOnProvidedExecutorTest() { Queue decodeTasks = new ConcurrentLinkedQueue<>(); - MessageDecoder decoder = new MessageDecoder(1000, decodeTasks::add, REGISTRY); + MessageDecoder decoder = new MessageDecoder(1000, decodeTasks::add, REGISTRY, NOOP_METRICS); AtomicInteger p1ready = new AtomicInteger(); AtomicInteger p2ready = new AtomicInteger(); @@ -458,7 +460,7 @@ public OutputStream encode(OutputStream byteArrayOutputStream) throws IOExceptio } }); - MessageDecoder decoder = new MessageDecoder(300, Runnable::run, new CodecRegistry(codecs)); + MessageDecoder decoder = new MessageDecoder(300, Runnable::run, new CodecRegistry(codecs), NOOP_METRICS); AtomicInteger ready = new AtomicInteger(); ReadPartitionDecoder p1 = new ReadPartitionDecoder("p1", decoder, PS1, null, ready::incrementAndGet); @@ -522,7 +524,7 @@ public void wrongDecoderTest() { throw new RejectedExecutionException("rejected"); }; - MessageDecoder decoder = new MessageDecoder(200, decompressor, REGISTRY); + MessageDecoder decoder = new MessageDecoder(200, decompressor, REGISTRY, NOOP_METRICS); AtomicInteger ready = new AtomicInteger(); ReadPartitionDecoder p1 = new ReadPartitionDecoder("p1", decoder, PS1, null, ready::incrementAndGet);