Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions core/src/main/java/tech/ydb/core/grpc/GrpcTransport.java
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ <ReqT, RespT> GrpcReadWriteStream<RespT, ReqT> readWriteStreamCall(

String getDatabase();

/**
* Returns the configured discovery endpoint as host:port, or an empty string if unavailable.
* @return configured discovery endpoint
*/
default String getEndpoint() {
return "";
}

ScheduledExecutorService getScheduler();

default Tracer getTracer() {
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/tech/ydb/core/impl/BaseGrpcTransport.java
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ protected BaseGrpcTransport(EndpointRecord serverEndpoint, String buildInfo) {

protected abstract AuthCallOptions getAuthCallOptions();

@Override
public String getEndpoint() {
return serverEndpoint.getHostAndPort();
}

protected abstract GrpcChannel getChannel(GrpcRequestSettings settings);

protected void pessimizeEndpoint(EndpointRecord endpoint, String reason) {
Expand Down
8 changes: 8 additions & 0 deletions topic/src/main/java/tech/ydb/topic/TopicRpc.java
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,12 @@ default GrpcReadWriteStream<StreamWriteMessage.FromServer, StreamWriteMessage.Fr
GrpcReadWriteStream<StreamReadMessage.FromServer, StreamReadMessage.FromClient> readSession(String traceId);

ScheduledExecutorService getScheduler();

default String getEndpoint() {
return "";
}

default String getDatabase() {
return "";
}
}
10 changes: 10 additions & 0 deletions topic/src/main/java/tech/ydb/topic/impl/GrpcTopicRpc.java
Original file line number Diff line number Diff line change
Expand Up @@ -130,4 +130,14 @@ public GrpcReadWriteStream<StreamReadMessage.FromServer, StreamReadMessage.FromC
public ScheduledExecutorService getScheduler() {
return transport.getScheduler();
}

@Override
public String getEndpoint() {
return transport.getEndpoint();
}

@Override
public String getDatabase() {
return transport.getDatabase();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ public AsyncReaderImpl(TopicRpc topicRpc,
this.decompressor = new LazyExecutor("reader[" + debugId + "]-decoder", settings.getDecompressionExecutor());
this.controlEventsExecutor = new SerialExecutor(processor);

this.config = new ReadConfig(codecRegistry, controlEventsExecutor, processor, decompressor, settings);
this.config = new ReadConfig(codecRegistry, controlEventsExecutor, processor, decompressor, settings,
topicRpc.getEndpoint(), topicRpc.getDatabase());
this.impl = new Impl(topicRpc, debugId, settings, config);

String readerName = settings.getReaderName();
Expand Down
8 changes: 7 additions & 1 deletion topic/src/main/java/tech/ydb/topic/read/impl/ReadConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,20 @@ public class ReadConfig {

public ReadConfig(CodecRegistry codecRegistry, Executor control, Executor processor, Executor decompressor,
ReaderSettings settings) {
this(codecRegistry, control, processor, decompressor, settings, "", "");
}

public ReadConfig(CodecRegistry codecRegistry, Executor control, Executor processor, Executor decompressor,
ReaderSettings settings, String endpoint, String database) {
this.codecRegistry = codecRegistry;
this.control = control;
this.processor = processor;
this.decompressor = decompressor;
this.consumerName = settings.getConsumerName();
this.maxMemoryUsageBytes = settings.getMaxMemoryUsageBytes();
this.maxBatchSize = settings.getMaxBatchSize();
this.metrics = new ReaderMetrics(settings.getMeter(), settings.getConsumerName(), settings.getReaderName());
this.metrics = new ReaderMetrics(settings.getMeter(), settings.getConsumerName(), settings.getReaderName(),
endpoint, database);
}

public CodecRegistry getCodecRegistry() {
Expand Down
20 changes: 15 additions & 5 deletions topic/src/main/java/tech/ydb/topic/read/impl/ReaderMetrics.java
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ interface Source {
private MetricRegistration ageGauge = MetricRegistration.NOOP;

ReaderMetrics(Meter meter, String consumer, String readerName) {
this(meter, consumer, readerName, "", "");
}

ReaderMetrics(Meter meter, String consumer, String readerName, String endpoint, String database) {
this.meter = meter;
this.enabled = meter != Meter.NOOP;
this.deliveredMessages = meter.createCounter(
Expand All @@ -67,7 +71,8 @@ interface Source {
"The time spent decompressing a message, excluding queueing.");
this.processingDuration = meter.createHistogram("ydb.topic.reader.message.processing.duration", "s",
"The time from receipt until a message is ready for delivery.");
this.commonAttributes = createCommonAttributes(consumer, readerName);
this.commonAttributes = enabled
? createCommonAttributes(consumer, readerName, endpoint, database) : new Attr[0];
}

void addSource(Source source) {
Expand Down Expand Up @@ -228,15 +233,20 @@ private void record(LongCounter counter, long value, Attr... extraAttributes) {
}
}

private static Attr[] createCommonAttributes(String consumer, String readerName) {
private static Attr[] createCommonAttributes(String consumer, String readerName, String endpoint, String database) {
Attr endpointAttr = Attr.of("endpoint", endpoint);
Attr databaseAttr = Attr.of("database", database);
boolean hasReaderName = readerName != null && !readerName.isEmpty();
if (consumer == null) {
return hasReaderName ? new Attr[]{Attr.of("reader.name", readerName)} : new Attr[0];
return hasReaderName
? new Attr[]{endpointAttr, databaseAttr, Attr.of("reader.name", readerName)}
: new Attr[]{endpointAttr, databaseAttr};
}
if (!hasReaderName) {
return new Attr[]{Attr.of("consumer", consumer)};
return new Attr[]{endpointAttr, databaseAttr, Attr.of("consumer", consumer)};
}
return new Attr[]{Attr.of("consumer", consumer), Attr.of("reader.name", readerName)};
return new Attr[]{
endpointAttr, databaseAttr, Attr.of("consumer", consumer), Attr.of("reader.name", readerName)};
}

private static class CommitOffsets {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ public SyncReaderImpl(TopicRpc topicRpc, ReaderSettings settings, @Nonnull Codec
this.debugId = DebugTools.createDebugId(settings.getLogPrefix());
this.decompressor = new LazyExecutor("reader[" + debugId + "]-decoder", settings.getDecompressionExecutor());

this.config = new ReadConfig(codecRegistry, Runnable::run, Runnable::run, decompressor, settings);
this.config = new ReadConfig(codecRegistry, Runnable::run, Runnable::run, decompressor, settings,
topicRpc.getEndpoint(), topicRpc.getDatabase());
this.impl = new Impl(topicRpc, debugId, settings, config);

String readerName = settings.getReaderName();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ public WriterImpl(TopicRpc topicRpc, WriterSettings settings, Executor compressi
public WriterImpl(TopicRpc topicRpc, WriteStreamFactory factory, WriterSettings settings,
Executor compressionExecutor, @Nonnull CodecRegistry codecRegistry) {
this.debugId = DebugTools.createDebugId(settings.getLogPrefix());
this.metrics = new WriterMetrics(settings.getMeter(), settings.getTopicPath(), settings.getWriterName());
this.metrics = new WriterMetrics(settings.getMeter(), settings.getTopicPath(), settings.getWriterName(),
topicRpc.getEndpoint(), topicRpc.getDatabase());
this.stream = new WriteSession(debugId, factory, settings, topicRpc.getScheduler(), new ListenerImpl());
this.writeQueue = new WriterQueue(debugId, settings, metrics, codecRegistry, compressionExecutor, sendTask);
Observability.reportMetricsUsage(settings.getMeter());
Expand Down
11 changes: 8 additions & 3 deletions topic/src/main/java/tech/ydb/topic/write/impl/WriterMetrics.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ final class WriterMetrics {
private MetricRegistration oldestAgeGauge = MetricRegistration.NOOP;

WriterMetrics(Meter meter, String topic, String writerName) {
this(meter, topic, writerName, "", "");
}

WriterMetrics(Meter meter, String topic, String writerName, String endpoint, String database) {
this.meter = meter;
this.enabled = meter != Meter.NOOP;
this.sendingMessages = meter.createCounter("ydb.topic.writer.sending.messages", MESSAGE_UNIT,
Expand All @@ -52,9 +56,10 @@ final class WriterMetrics {
"The time spent compressing a message, excluding queueing.");
this.processingDuration = meter.createHistogram("ydb.topic.writer.message.processing.duration", "s",
"The time from buffer acceptance until a message is ready for sending.");
this.commonAttributes = writerName == null
? new Attr[0]
: new Attr[]{Attr.of("topic", topic), Attr.of("writer.name", writerName)};
this.commonAttributes = !enabled ? new Attr[0] : writerName == null
? new Attr[]{Attr.of("endpoint", endpoint), Attr.of("database", database)}
: new Attr[]{Attr.of("endpoint", endpoint), Attr.of("database", database),
Attr.of("topic", topic), Attr.of("writer.name", writerName)};
}

void register(LongSupplier bufferUsed, LongSupplier bufferLimit, LongSupplier oldestTimestamp) {
Expand Down
Loading