From 472f090332185e0753927473b3550fbf6f75c932 Mon Sep 17 00:00:00 2001 From: nastya Date: Tue, 22 Sep 2026 14:19:03 +0300 Subject: [PATCH 1/5] Add topic reader names to logs --- CHANGELOG.md | 2 + docs/topic.rst | 5 + examples/topic/reader_async_example.py | 6 +- examples/topic/reader_example.py | 6 +- ydb/_grpc/grpcwrapper/ydb_topic.py | 3 + ydb/_grpc/grpcwrapper/ydb_topic_test.py | 24 ++- ydb/_topic_reader/topic_reader.py | 8 +- ydb/_topic_reader/topic_reader_asyncio.py | 91 +++++++--- .../topic_reader_asyncio_test.py | 156 +++++++++++++++++- ydb/_topic_reader/topic_reader_sync.py | 7 +- ydb/topic.py | 2 + 11 files changed, 268 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a1c411f..9b69adc41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Add optional topic reader names for identifying logical readers in server diagnostics and SDK logs + ## 3.33.2 ## * Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried diff --git a/docs/topic.rst b/docs/topic.rst index bd5455691..999d8733c 100644 --- a/docs/topic.rst +++ b/docs/topic.rst @@ -378,8 +378,13 @@ Reader Parameters consumer="my-consumer", buffer_size_bytes=50 * 1024 * 1024, # client-side buffer (default: 50 MB) buffer_release_threshold=0.5, # see below (default: 0.5) + reader_name="payments-worker", # optional diagnostic name ) +``reader_name`` identifies a logical reader in server-side diagnostics and SDK logs. If it is +omitted or empty, the SDK generates a process-local name in the form ``reader-N``. The name +remains unchanged when the reader reconnects. Explicit names do not have to be unique. + ``buffer_size_bytes`` controls how many bytes the server is allowed to send before the client signals that it is ready for more. The server will not exceed this limit. diff --git a/examples/topic/reader_async_example.py b/examples/topic/reader_async_example.py index 71e774555..464293c1a 100644 --- a/examples/topic/reader_async_example.py +++ b/examples/topic/reader_async_example.py @@ -9,7 +9,11 @@ async def connect(): connection_string="grpc://localhost:2135?database=/local", credentials=ydb.credentials.AnonymousCredentials(), ) - reader = db.topic_client.reader("/local/topic", consumer="consumer") + reader = db.topic_client.reader( + "/local/topic", + consumer="consumer", + reader_name="payments-worker", + ) return reader diff --git a/examples/topic/reader_example.py b/examples/topic/reader_example.py index ab8157a9b..53caab4bd 100644 --- a/examples/topic/reader_example.py +++ b/examples/topic/reader_example.py @@ -8,7 +8,11 @@ def connect(): connection_string="grpc://localhost:2135?database=/local", credentials=ydb.credentials.AnonymousCredentials(), ) - reader = db.topic_client.reader("/local/topic", consumer="consumer") + reader = db.topic_client.reader( + "/local/topic", + consumer="consumer", + reader_name="payments-worker", + ) return reader diff --git a/ydb/_grpc/grpcwrapper/ydb_topic.py b/ydb/_grpc/grpcwrapper/ydb_topic.py index 19b38bf01..0de3a7495 100644 --- a/ydb/_grpc/grpcwrapper/ydb_topic.py +++ b/ydb/_grpc/grpcwrapper/ydb_topic.py @@ -486,11 +486,14 @@ class InitRequest(IToProto): topics_read_settings: List["StreamReadMessage.InitRequest.TopicReadSettings"] consumer: Optional[str] auto_partitioning_support: bool + reader_name: Optional[str] = None def to_proto(self) -> ydb_topic_pb2.StreamReadMessage.InitRequest: res = ydb_topic_pb2.StreamReadMessage.InitRequest() if self.consumer is not None: res.consumer = self.consumer + if self.reader_name is not None: + res.reader_name = self.reader_name for settings in self.topics_read_settings: res.topics_read_settings.append(settings.to_proto()) res.auto_partitioning_support = self.auto_partitioning_support diff --git a/ydb/_grpc/grpcwrapper/ydb_topic_test.py b/ydb/_grpc/grpcwrapper/ydb_topic_test.py index b9e306034..5e6ff1c78 100644 --- a/ydb/_grpc/grpcwrapper/ydb_topic_test.py +++ b/ydb/_grpc/grpcwrapper/ydb_topic_test.py @@ -2,7 +2,7 @@ from google.protobuf.json_format import MessageToDict -from ydb._grpc.grpcwrapper.ydb_topic import OffsetsRange +from ydb._grpc.grpcwrapper.ydb_topic import OffsetsRange, StreamReadMessage from .ydb_topic import AlterTopicRequest from .ydb_topic_public_types import ( AlterTopicRequestParams, @@ -96,3 +96,25 @@ def test_alter_topic_request_from_public_to_proto(): } assert msg_dict == expected_dict + + +def test_stream_read_init_request_serializes_reader_name(): + request = StreamReadMessage.InitRequest( + topics_read_settings=[], + consumer="analytics", + auto_partitioning_support=True, + reader_name="payments-worker", + ) + + assert request.to_proto().reader_name == "payments-worker" + + +def test_stream_read_init_request_omits_reader_name_by_default(): + request = StreamReadMessage.InitRequest( + topics_read_settings=[], + consumer="analytics", + auto_partitioning_support=True, + ) + + assert request.reader_name is None + assert request.to_proto().reader_name == "" diff --git a/ydb/_topic_reader/topic_reader.py b/ydb/_topic_reader/topic_reader.py index e4545f3a6..59f280d59 100644 --- a/ydb/_topic_reader/topic_reader.py +++ b/ydb/_topic_reader/topic_reader.py @@ -60,13 +60,18 @@ class PublicReaderSettings: buffer_release_threshold: float = 0.5 """Min fraction of buffer_size_bytes to accumulate before sending a new ReadRequest (0.0 = immediately after every batch).""" + reader_name: Optional[str] = None + """Optional reader name used to identify this logical reader in diagnostics.""" + def __post_init__(self): + if self.reader_name is not None and not isinstance(self.reader_name, str): + raise TypeError("Unsupported type for reader_name field: '%s'" % type(self.reader_name)) if not (0.0 <= self.buffer_release_threshold <= 1.0): raise ValueError("buffer_release_threshold must be in [0.0, 1.0], got %s" % self.buffer_release_threshold) # check possible create init message _ = self._init_message() - def _init_message(self) -> StreamReadMessage.InitRequest: + def _init_message(self, *, reader_name: Optional[str] = None) -> StreamReadMessage.InitRequest: if self.consumer is not None and not isinstance(self.consumer, str): raise TypeError("Unsupported type for customer field: '%s'" % type(self.consumer)) @@ -87,6 +92,7 @@ def _init_message(self) -> StreamReadMessage.InitRequest: topics_read_settings=list(map(PublicTopicSelector._to_topic_read_settings, selectors)), # type: ignore consumer=self.consumer, auto_partitioning_support=self.auto_partitioning_support, + reader_name=self.reader_name if reader_name is None else reader_name, ) def _retry_settings(self) -> RetrySettings: diff --git a/ydb/_topic_reader/topic_reader_asyncio.py b/ydb/_topic_reader/topic_reader_asyncio.py index e5c9f3a59..11c56ef85 100644 --- a/ydb/_topic_reader/topic_reader_asyncio.py +++ b/ydb/_topic_reader/topic_reader_asyncio.py @@ -87,6 +87,7 @@ def __init__(self): class PublicAsyncIOReader: _loop: asyncio.AbstractEventLoop _closed: bool + _log_prefix: str _settings: topic_reader.PublicReaderSettings _reconnector: ReaderReconnector _parent: typing.Any # need for prevent close parent client by GC @@ -100,8 +101,10 @@ def __init__( ): self._loop = asyncio.get_running_loop() self._closed = False + self._log_prefix = "topic reader" self._settings = settings self._reconnector = ReaderReconnector(driver, settings, self._loop) + self._log_prefix = self._reconnector._log_prefix self._parent = _parent async def __aenter__(self): @@ -111,13 +114,19 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close() def __del__(self): - if not self._closed: - try: - logger.debug("Topic reader was not closed properly. Consider using method close().") - task = self._loop.create_task(self.close(flush=False)) - task.set_name("close reader") - except BaseException: - logger.warning("Something went wrong during reader close in __del__") + if getattr(self, "_closed", True): + return + + loop = getattr(self, "_loop", None) + if getattr(self, "_reconnector", None) is None or loop is None or loop.is_closed() or not loop.is_running(): + return + + try: + logger.debug("%s was not closed properly. Consider using method close().", self._log_prefix) + task = loop.create_task(self.close(flush=False)) + task.set_name("close reader") + except BaseException: + logger.warning("%s failed to close in __del__", self._log_prefix) async def wait_message(self): """ @@ -140,7 +149,7 @@ async def receive_batch( use asyncio.wait_for for wait with timeout. """ - logger.debug("receive_batch max_messages=%s max_bytes=%s", max_messages, max_bytes) + logger.debug("%s receive_batch max_messages=%s max_bytes=%s", self._log_prefix, max_messages, max_bytes) await self._reconnector.wait_message() return self._reconnector.receive_batch_nowait( max_messages=max_messages, @@ -163,7 +172,13 @@ async def receive_batch_with_tx( use asyncio.wait_for for wait with timeout. """ - logger.debug("receive_batch_with_tx tx=%s max_messages=%s max_bytes=%s", tx, max_messages, max_bytes) + logger.debug( + "%s receive_batch_with_tx tx=%s max_messages=%s max_bytes=%s", + self._log_prefix, + tx, + max_messages, + max_bytes, + ) await self._reconnector.wait_message() return self._reconnector.receive_batch_with_tx_nowait( tx=tx, @@ -177,7 +192,7 @@ async def receive_message(self) -> typing.Optional[datatypes.PublicMessage]: use asyncio.wait_for for wait with timeout. """ - logger.debug("receive_message") + logger.debug("%s receive_message", self._log_prefix) await self._reconnector.wait_message() return self._reconnector.receive_message_nowait() @@ -188,7 +203,7 @@ def commit(self, batch: typing.Union[datatypes.PublicMessage, datatypes.PublicBa For the method no way check the commit result (for example if lost connection - commits will not re-send and committed messages will receive again). """ - logger.debug("commit message or batch") + logger.debug("%s commit message or batch", self._log_prefix) if self._settings.consumer is None: raise issues.Error("Commit operations are not supported for topic reader without consumer.") @@ -207,7 +222,7 @@ async def commit_with_ack(self, batch: typing.Union[datatypes.PublicMessage, dat before receive commit ack. Message may be acked or not (if not - it will send in other read session, to this or other reader). """ - logger.debug("commit_with_ack message or batch") + logger.debug("%s commit_with_ack message or batch", self._log_prefix) if self._settings.consumer is None: raise issues.Error("Commit operations are not supported for topic reader without consumer.") @@ -218,10 +233,10 @@ async def close(self, flush: bool = True): if self._closed: raise TopicReaderClosedError() - logger.debug("Close topic reader") + logger.debug("%s close", self._log_prefix) self._closed = True await self._reconnector.close(flush) - logger.debug("Topic reader was closed") + logger.debug("%s was closed", self._log_prefix) @property def read_session_id(self) -> Optional[str]: @@ -235,6 +250,8 @@ class ReaderReconnector: _settings: topic_reader.PublicReaderSettings _driver: Driver _background_tasks: Set[Task] + _reader_name: str + _log_prefix: str _state_changed: asyncio.Event _stream_reader: Optional["ReaderStream"] @@ -251,9 +268,11 @@ def __init__( self._id = ReaderReconnector._static_reader_reconnector_counter.inc_and_get() self._settings = settings self._driver = driver + self._reader_name = settings.reader_name or "reader-%d" % self._id + self._log_prefix = "topic reader reader_name=%r reader_id=%s" % (self._reader_name, self._id) self._loop = loop if loop is not None else asyncio.get_running_loop() self._background_tasks = set() - logger.debug("init reader reconnector id=%s", self._id) + logger.debug("%s initialize reconnector", self._log_prefix) self._state_changed = asyncio.Event() self._stream_reader = None @@ -269,21 +288,30 @@ async def _connection_loop(self): if self._closed: return try: - logger.debug("reader %s connect attempt %s", self._id, attempt) - self._stream_reader = await ReaderStream.create(self._id, self._driver, self._settings) - logger.debug("reader %s connected stream %s", self._id, self._stream_reader._id) + logger.debug("%s connect attempt=%s", self._log_prefix, attempt) + self._stream_reader = await ReaderStream.create( + self._id, + self._driver, + self._settings, + reader_name=self._reader_name, + ) + logger.debug("%s connected stream_id=%s", self._log_prefix, self._stream_reader._id) attempt = 0 self._state_changed.set() await self._stream_reader.wait_error() except BaseException as err: - logger.debug("reader %s, attempt %s connection loop error %s", self._id, attempt, err) + logger.debug("%s connection attempt=%s failed: %s", self._log_prefix, attempt, err) retry_info = check_retriable_error(err, self._settings._retry_settings(), attempt) if not retry_info.is_retriable: - logger.debug("reader %s stop connection loop due to %s", self._id, err) + logger.debug("%s stop connection loop: %s", self._log_prefix, err) self._set_first_error(err) return - logger.debug("sleep before retry for %s seconds", retry_info.sleep_timeout_seconds) + logger.debug( + "%s sleep before retry for %s seconds", + self._log_prefix, + retry_info.sleep_timeout_seconds, + ) await asyncio.sleep(retry_info.sleep_timeout_seconds) @@ -465,7 +493,7 @@ def commit(self, batch: datatypes.ICommittable) -> datatypes.PartitionSession.Co return self._stream_reader.commit(batch) async def close(self, flush: bool): - logger.debug("reader reconnector %s close", self._id) + logger.debug("%s close reconnector", self._log_prefix) # Mark closed so the connection loop won't start a new stream, then close the # current stream with the requested flush before cancelling the loop. On a normal # close this flushes pending commits; cancelling the loop first would let it close @@ -506,6 +534,7 @@ class ReaderStream: _loop: asyncio.AbstractEventLoop _id: int _reader_reconnector_id: int + _reader_name: str _session_id: str _stream: Optional[IGrpcWrapperAsyncIO] _started: bool @@ -537,12 +566,16 @@ def __init__( reader_reconnector_id: int, settings: topic_reader.PublicReaderSettings, get_token_function: Optional[Callable[[], str]] = None, + *, + reader_name: str, ): self._loop = asyncio.get_running_loop() self._id = ReaderStream._static_id_counter.inc_and_get() self._reader_reconnector_id = reader_reconnector_id + self._reader_name = reader_name self._session_id = "not initialized" - self._log_prefix = "reader %s stream %s session=%s" % ( + self._log_prefix = "topic reader reader_name=%r reader_id=%s stream_id=%s session_id=%s" % ( + self._reader_name, self._reader_reconnector_id, self._id, self._session_id, @@ -572,13 +605,15 @@ def __init__( self._settings = settings - logger.debug("created ReaderStream id=%s reconnector=%s", self._id, self._reader_reconnector_id) + logger.debug("%s created", self._log_prefix) @staticmethod async def create( reader_reconnector_id: int, driver: SupportedDriverType, settings: topic_reader.PublicReaderSettings, + *, + reader_name: str, ) -> "ReaderStream": stream = GrpcWrapperAsyncIO(StreamReadMessage.FromServer.from_proto) reader = None @@ -590,8 +625,9 @@ async def create( reader_reconnector_id, settings, get_token_function=creds.get_auth_token if creds else None, + reader_name=reader_name, ) - await reader._start(stream, settings._init_message()) + await reader._start(stream, settings._init_message(reader_name=reader_name)) except BaseException: # If create() is interrupted (e.g. reader.close() cancels the connection loop # mid-reconnect) the in-flight stream is not yet assigned to the reconnector, so @@ -623,7 +659,8 @@ async def _start(self, stream: IGrpcWrapperAsyncIO, init_message: StreamReadMess if isinstance(init_response.server_message, StreamReadMessage.InitResponse): self._session_id = init_response.server_message.session_id - self._log_prefix = "reader %s stream %s session=%s" % ( + self._log_prefix = "topic reader reader_name=%r reader_id=%s stream_id=%s session_id=%s" % ( + self._reader_name, self._reader_reconnector_id, self._id, self._session_id, @@ -820,7 +857,7 @@ async def _read_messages_loop(self): "Unexpected message in _read_messages_loop: %s" % type(message.server_message) ) except issues.UnexpectedGrpcMessage as e: - logger.exception("unexpected message in stream reader: %s" % e) + logger.exception("%s unexpected message in stream reader: %s", self._log_prefix, e) self._state_changed.set() except asyncio.CancelledError as e: diff --git a/ydb/_topic_reader/topic_reader_asyncio_test.py b/ydb/_topic_reader/topic_reader_asyncio_test.py index c018604f2..989d13181 100644 --- a/ydb/_topic_reader/topic_reader_asyncio_test.py +++ b/ydb/_topic_reader/topic_reader_asyncio_test.py @@ -88,6 +88,20 @@ def stub_message(id: int): ) +def test_partially_initialized_public_reader_does_not_schedule_close(): + reader = object.__new__(topic_reader_asyncio.PublicAsyncIOReader) + reader._closed = False + reader._log_prefix = "topic reader" + reader._loop = mock.Mock() + reader._loop.is_closed.return_value = False + reader._loop.is_running.return_value = True + + reader.__del__() + + reader._loop.create_task.assert_not_called() + reader._closed = True + + @pytest.fixture() def default_reader_settings(default_executor): return PublicReaderSettings( @@ -155,6 +169,7 @@ def second_partition_session(self, default_reader_settings, stream_reader_starte return stream_reader_started._partition_sessions[partition_session.id] async def get_started_reader(self, stream, *args, **kwargs) -> ReaderStream: + kwargs.setdefault("reader_name", "reader-%d" % self.default_reader_reconnector_id) reader = ReaderStream(self.default_reader_reconnector_id, *args, **kwargs) init_message = object() @@ -182,6 +197,20 @@ async def get_started_reader(self, stream, *args, **kwargs) -> ReaderStream: return reader + async def test_log_prefix_identifies_reader_stream_and_session(self, stream, default_reader_settings): + reader = await self.get_started_reader( + stream, + default_reader_settings, + reader_name="payments worker\nprimary", + ) + + assert reader._log_prefix == ( + "topic reader reader_name='payments worker\\nprimary' " + "reader_id=%s stream_id=%s session_id=test-session" % (self.default_reader_reconnector_id, reader._id) + ) + + await reader.close(False) + @pytest.fixture() async def stream_reader_started(self, stream, default_reader_settings) -> ReaderStream: return await self.get_started_reader(stream, default_reader_settings) @@ -601,7 +630,11 @@ async def test_error_from_status_code(self, stream, stream_reader_finish_with_er stream_reader_finish_with_error.receive_batch_nowait() async def test_init_reader(self, stream, default_reader_settings): - reader = ReaderStream(self.default_reader_reconnector_id, default_reader_settings) + reader = ReaderStream( + self.default_reader_reconnector_id, + default_reader_settings, + reader_name="reader-%d" % self.default_reader_reconnector_id, + ) init_message = StreamReadMessage.InitRequest( consumer="test-consumer", topics_read_settings=[ @@ -1549,7 +1582,11 @@ def logged(): async def test_init_timeout_parameter(self, stream, default_reader_settings): """Test that ReaderStream._start calls stream.receive with timeout=10""" - reader = ReaderStream(self.default_reader_reconnector_id, default_reader_settings) + reader = ReaderStream( + self.default_reader_reconnector_id, + default_reader_settings, + reader_name="reader-%d" % self.default_reader_reconnector_id, + ) init_message = default_reader_settings._init_message() # Mock stream.receive to check if timeout is passed @@ -1568,7 +1605,11 @@ async def test_init_timeout_parameter(self, stream, default_reader_settings): async def test_init_timeout_behavior(self, stream, default_reader_settings): """Test that ReaderStream._start raises TopicReaderError when receive times out""" - reader = ReaderStream(self.default_reader_reconnector_id, default_reader_settings) + reader = ReaderStream( + self.default_reader_reconnector_id, + default_reader_settings, + reader_name="reader-%d" % self.default_reader_reconnector_id, + ) init_message = default_reader_settings._init_message() # Mock stream.receive to directly raise TimeoutError when called with timeout @@ -1590,7 +1631,8 @@ async def timeout_receive(timeout=None): @pytest.mark.asyncio class TestReaderReconnector: - async def test_reconnect_on_repeatable_error(self, monkeypatch): + @pytest.mark.parametrize("configured_reader_name", [None, ""]) + async def test_reconnect_on_repeatable_error(self, monkeypatch, configured_reader_name): test_error = issues.Overloaded("test error") async def wait_error(): @@ -1620,8 +1662,11 @@ async def stream_create( reader_reconnector_id: int, driver: SupportedDriverType, settings: PublicReaderSettings, + *, + reader_name: str, ): nonlocal stream_index + reader_names.append(reader_name) stream_index += 1 if stream_index == 1: return reader_stream_mock_with_error @@ -1630,12 +1675,19 @@ async def stream_create( else: raise Exception("unexpected create stream") + reader_names = [] with mock.patch.object(ReaderStream, "create", stream_create): - reconnector = ReaderReconnector(mock.Mock(), PublicReaderSettings("", "")) + reconnector = ReaderReconnector( + mock.Mock(), + PublicReaderSettings("", "", reader_name=configured_reader_name), + ) await wait_for_fast(reconnector.wait_message()) + assert reader_names == [reconnector._reader_name, reconnector._reader_name] + assert reconnector._reader_name == "reader-%d" % reconnector._id reader_stream_mock_with_error.wait_error.assert_any_await() reader_stream_mock_with_error.wait_messages.assert_any_await() + await reconnector.close(flush=False) async def test_close_during_reconnect_does_not_hang(self): # The connection loop must stop on reader.close() even while it is closing the old @@ -1668,7 +1720,7 @@ async def wait_forever(): create_calls = 0 - async def stream_create(reader_reconnector_id, driver, settings): + async def stream_create(reader_reconnector_id, driver, settings, *, reader_name): nonlocal create_calls create_calls += 1 return stream1 if create_calls == 1 else stream2 @@ -1702,7 +1754,14 @@ async def start(self, driver, stub, method): with mock.patch.object(topic_reader_asyncio, "GrpcWrapperAsyncIO", FakeStream): # Real create(); no InitResponse is sent, so it parks inside _start() on # `await stream.receive()` (the only reachable cancellation point in create()). - create_task = asyncio.create_task(ReaderStream.create(7, driver, default_reader_settings)) + create_task = asyncio.create_task( + ReaderStream.create( + 7, + driver, + default_reader_settings, + reader_name="reader-7", + ) + ) await wait_condition(lambda: bool(built) and not built[0].from_client.empty()) assert not create_task.done() @@ -1732,7 +1791,7 @@ async def receive(timeout=None): stream.write = mock.Mock() stream.close = mock.Mock() - reader = ReaderStream(4, default_reader_settings) + reader = ReaderStream(4, default_reader_settings, reader_name="reader-4") await reader._start(stream, default_reader_settings._init_message()) # Bug: wait_error() hangs forever because _first_error is never set. @@ -1766,7 +1825,11 @@ async def _get_started_reader(self, stream, threshold, buffer_size_bytes, defaul buffer_release_threshold=threshold, decoder_executor=default_executor, ) - reader = ReaderStream(self.default_reader_reconnector_id, settings) + reader = ReaderStream( + self.default_reader_reconnector_id, + settings, + reader_name="reader-%d" % self.default_reader_reconnector_id, + ) init_message = object() start = asyncio.create_task(reader._start(stream, init_message)) @@ -1930,3 +1993,78 @@ async def test_threshold_one_flushes_when_bytes_match_buffer_size(self, stream, assert msg.client_message.bytes_size == 1000 await reader.close(False) + + +def test_reader_settings_forward_reader_name(): + settings = PublicReaderSettings( + consumer="analytics", + topic="/Root/events", + reader_name="payments-worker", + ) + + assert settings._init_message().reader_name == "payments-worker" + assert settings._init_message(reader_name="reader-42").to_proto().reader_name == "reader-42" + + +def test_reader_settings_reject_non_string_reader_name(): + with pytest.raises(TypeError, match="reader_name"): + PublicReaderSettings( + consumer="analytics", + topic="/Root/events", + reader_name=42, + ) + + +@pytest.mark.asyncio +async def test_reader_reconnector_uses_configured_or_process_local_reader_name(): + async def wait_forever(self): + await asyncio.Future() + + with mock.patch.object(ReaderReconnector, "_connection_loop", wait_forever): + generated_from_none = ReaderReconnector( + mock.Mock(), + PublicReaderSettings("analytics", "/Root/events"), + ) + generated_from_empty = ReaderReconnector( + mock.Mock(), + PublicReaderSettings("analytics", "/Root/events", reader_name=""), + ) + configured = ReaderReconnector( + mock.Mock(), + PublicReaderSettings("analytics", "/Root/events", reader_name="payments-worker"), + ) + + assert generated_from_none._reader_name == "reader-%d" % generated_from_none._id + assert generated_from_empty._reader_name == "reader-%d" % generated_from_empty._id + assert generated_from_none._reader_name != generated_from_empty._reader_name + assert configured._reader_name == "payments-worker" + assert configured._log_prefix == "topic reader reader_name='payments-worker' reader_id=%s" % configured._id + + await asyncio.gather( + generated_from_none.close(flush=False), + generated_from_empty.close(flush=False), + configured.close(flush=False), + ) + + +@pytest.mark.parametrize( + "client_class, reader_class_name", [("TopicClient", "TopicReader"), ("TopicClientAsyncIO", "TopicReaderAsyncIO")] +) +def test_topic_clients_forward_reader_name(client_class, reader_class_name): + from .. import topic as topic_module + + client = object.__new__(getattr(topic_module, client_class)) + client._closed = False + client._driver = mock.Mock() + client._executor = mock.Mock() + + with mock.patch.object(topic_module, reader_class_name, return_value=mock.sentinel.reader) as reader_class: + result = client.reader( + "/Root/events", + consumer="analytics", + reader_name="payments-worker", + ) + + assert result is mock.sentinel.reader + assert reader_class.call_args.args[1].reader_name == "payments-worker" + client._closed = True diff --git a/ydb/_topic_reader/topic_reader_sync.py b/ydb/_topic_reader/topic_reader_sync.py index a2a24fef6..4a09ebdf3 100644 --- a/ydb/_topic_reader/topic_reader_sync.py +++ b/ydb/_topic_reader/topic_reader_sync.py @@ -32,6 +32,7 @@ class TopicReaderSync: _caller: CallFromSyncToAsync _async_reader: PublicAsyncIOReader _closed: bool + _log_prefix: str _settings: PublicReaderSettings _parent: typing.Any # need for prevent stop the client by GC @@ -44,6 +45,7 @@ def __init__( _parent=None, # need for prevent stop the client by GC ): self._closed = False + self._log_prefix = "topic reader" if eventloop: loop = eventloop @@ -56,6 +58,7 @@ async def create_reader(): return PublicAsyncIOReader(driver, settings) self._async_reader = asyncio.run_coroutine_threadsafe(create_reader(), loop).result() + self._log_prefix = self._async_reader._log_prefix self._settings = settings @@ -64,10 +67,10 @@ async def create_reader(): def __del__(self): if not self._closed: try: - logger.debug("Topic reader was not closed properly. Consider using method close().") + logger.debug("%s was not closed properly. Consider using method close().", self._log_prefix) self.close(flush=False) except BaseException: - logger.warning("Something went wrong during reader close in __del__") + logger.warning("%s failed to close in __del__", self._log_prefix) def __enter__(self): return self diff --git a/ydb/topic.py b/ydb/topic.py index e76956abc..2a588c25c 100644 --- a/ydb/topic.py +++ b/ydb/topic.py @@ -295,6 +295,7 @@ def reader( auto_partitioning_support: Optional[bool] = True, # Auto partitioning feature flag. Default - True. event_handler: Optional[TopicReaderEvents.EventHandler] = None, buffer_release_threshold: float = 0.5, + reader_name: Optional[str] = None, ) -> TopicReaderAsyncIO: logger.debug("Create reader for topic=%s consumer=%s", topic, consumer) @@ -631,6 +632,7 @@ def reader( auto_partitioning_support: Optional[bool] = True, # Auto partitioning feature flag. Default - True. event_handler: Optional[TopicReaderEvents.EventHandler] = None, buffer_release_threshold: float = 0.5, + reader_name: Optional[str] = None, ) -> TopicReader: logger.debug("Create reader for topic=%s consumer=%s", topic, consumer) if not decoder_executor: From 88d630eed6678491111b2e4f908520b8e0119c69 Mon Sep 17 00:00:00 2001 From: nastya Date: Tue, 27 Oct 2026 17:14:56 +0300 Subject: [PATCH 2/5] Add topic reader metrics --- CHANGELOG.md | 1 + docs/observability.rst | 93 +- docs/opentelemetry.rst | 110 ++- docs/topic.rst | 5 +- examples/opentelemetry/README.md | 69 +- examples/opentelemetry/compose-e2e.yaml | 1 + examples/opentelemetry/otel_example.py | 34 +- setup.py | 2 +- tests/observability/test_metrics.py | 818 +++++++++++++++++- ydb/_topic_reader/topic_reader_asyncio.py | 113 ++- .../topic_reader_asyncio_test.py | 342 +++++++- ydb/observability/metrics.py | 270 +++++- ydb/opentelemetry/metrics_plugin.py | 41 +- 13 files changed, 1833 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b69adc41..97de48346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +* Add `ydb.topic.reader.received.messages`, reader-level `ydb.topic.reader.received.bytes`, and `ydb.topic.reader.delivered.messages` counters plus `ydb.topic.reader.local_buffer.messages`, `ydb.topic.reader.local_buffer.message_age.max`, and `ydb.topic.reader.partition_session.count` observable metrics for explicitly named topic readers; metrics-enabled clients now advertise `ydb-sdk-metrics/0.7.0` in `x-ydb-sdk-build-info` * Add optional topic reader names for identifying logical readers in server diagnostics and SDK logs ## 3.33.2 ## diff --git a/docs/observability.rst b/docs/observability.rst index f332cf7f8..f0b0bca92 100644 --- a/docs/observability.rst +++ b/docs/observability.rst @@ -268,17 +268,18 @@ Client-side metrics are enabled independently of tracing: disable_metrics() # turn metrics off — back to the no-op default Operation and retry metrics follow the currently active provider immediately. Call -``enable_metrics`` before creating query sessions or a ``QuerySessionPool`` whose -lifecycle should be instrumented. Session and pool lifecycle trackers created while -metrics are disabled keep shared no-op instrumentation and are not retrofitted later. +``enable_metrics`` before creating query sessions, a ``QuerySessionPool``, or a topic +Reader whose lifecycle should be instrumented. Session, pool, and Reader lifecycle +trackers created while metrics are disabled keep shared no-op instrumentation and are +not retrofitted later. Calling ``enable_metrics`` again replaces the active backend. New events go to the new backend, while operations and wait timers already in flight finish on the backend on which they started. Observable callbacks registered by an old backend become inactive. -Pools that were created while metrics were active retain their lightweight lifecycle -state across ``disable_metrics`` / re-enable, so a replacement backend observes their -current values. Pools created while metrics were disabled retain no-op lifecycle -instrumentation. +Pools and topic Readers that were created while metrics were active retain their +lightweight lifecycle state across ``disable_metrics`` / re-enable, so a replacement +backend observes their current values. Pools and Readers created while metrics were +disabled retain no-op lifecycle instrumentation. For OpenTelemetry the built-in convenience is ``ydb.opentelemetry.enable_metrics``, which builds an OTel-backed provider from a meter provider and installs it here — see @@ -352,6 +353,24 @@ adapter maps them to instruments on the ``"ydb.sdk"`` meter): * - ``ydb.client.retry.attempts`` - Histogram - Number of attempts performed for one logical retried operation. + * - ``ydb.topic.reader.received.messages`` + - Counter (``{message}``) + - Messages received by the SDK topic Reader before decoding. + * - ``ydb.topic.reader.received.bytes`` + - Counter (``By``) + - Total ``ReadResponse`` bytes received by the SDK topic Reader. + * - ``ydb.topic.reader.delivered.messages`` + - Counter (``{message}``) + - Messages returned by the topic Reader to the application. + * - ``ydb.topic.reader.local_buffer.messages`` + - ObservableUpDownCounter (``{message}``) + - Current number of messages owned by the topic Reader and not yet returned to the application. + * - ``ydb.topic.reader.local_buffer.message_age.max`` + - ObservableGauge (``s``) + - Age of the oldest message owned by the topic Reader and not yet returned to the application. + * - ``ydb.topic.reader.partition_session.count`` + - ObservableUpDownCounter (``{session}``) + - Current number of active partition sessions owned by the topic Reader. Attributes ~~~~~~~~~~ @@ -402,6 +421,54 @@ initial attach handshake does not count as closing an active session, and standa ``QuerySession`` instances do not publish pool metrics. A session publishes at most one closure event; the first terminal reason wins. +.. _topic-reader-receive-metrics: + +Topic Reader Metrics +~~~~~~~~~~~~~~~~~~~~ + +Topic Reader metrics carry ``endpoint``, ``database``, and +``reader.name``. The ``consumer`` attribute is present only for consumer-based reads; +the SDK omits it when reading without a consumer. ``ydb.topic.reader.received.messages``, +``ydb.topic.reader.delivered.messages``, and +``ydb.topic.reader.partition_session.count`` additionally carry ``topic`` because each +message and partition session has an unambiguous topic. The reader-level +``ydb.topic.reader.received.bytes``, ``ydb.topic.reader.local_buffer.messages``, and +``ydb.topic.reader.local_buffer.message_age.max`` metrics omit ``topic``: one +``ReadResponse`` can contain data from several topics, and the buffer gauges follow the +total retained by the logical Reader. Each response increments the byte counter once by +``ReadResponse.bytes_size``. +Metrics are published only when the application passes a non-empty ``reader_name`` to +``TopicClient.reader``. The name and metric context belong to the logical Reader and +remain stable across stream reconnects. The byte counter is updated as soon as the +transport delivers a ``ReadResponse`` to the SDK, before the response is converted to +batches; it is therefore counted even if conversion later fails. The message counter is +updated after the batches are created and queued, but before their payloads are decoded. +For a multi-topic Reader, message increments are +separated by ``topic`` while the response byte increment remains a single Reader-level +value. The delivery counter is updated only after a message or batch has been removed +from the SDK buffer and is ready to be returned by ``receive_message``, ``receive_batch``, +or ``receive_batch_with_tx``. A batch increments it by the number of messages actually +returned after applying ``max_messages`` and ``max_bytes``. Empty reads, timeouts, and +failures before the return point do not increment it. The local-buffer gauge increases +when accepted batches enter the decoding queue and decreases when messages leave the +internal buffer, including partial batch reads. It therefore includes both messages +waiting for decoding and decoded messages waiting for the application. +``local_buffer.message_age.max`` is computed with a monotonic clock from the receive +time of the oldest retained message; it is ``0`` when the buffer is empty. Receipt +cohorts are tracked internally per partition session so delivery from another partition +does not advance the oldest timestamp incorrectly. ``partition_session.count`` changes +when the server starts or stops a partition session, is split by ``topic``, and does not +carry a ``partition`` attribute. Each configured topic is observable with value ``0`` +even before its first partition session starts. Remaining messages and partition +sessions are reset when a stream closes or reconnects, and closing the logical Reader +removes its observations. Use rates of the cumulative counters to compare incoming and +application-visible traffic; read the observable instruments as current values. For +matching Reader attributes, buffer counts and partition-session counts are summed while +the maximum buffer age is published. + +See :ref:`opentelemetry-topic-reader-metrics` for an OpenTelemetry setup example. A +complete runnable version is available in ``examples/opentelemetry/otel_example.py``. + Writing a Custom Metrics Backend ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -443,12 +510,14 @@ subclass; just match the signatures: The instrument kind for each metric is implied by which method the SDK calls: names in the *Histogram* rows above arrive through ``record``, *Counter* / *UpDownCounter* names -through ``add``, and the three *ObservableUpDownCounter* gauges -(``ydb.query.session.count`` / ``max`` / ``min``) through ``observe_gauge``. A backend +through ``add``, and the six observable instruments (the three query-session gauges, +``ydb.topic.reader.local_buffer.messages``, +``ydb.topic.reader.local_buffer.message_age.max``, and +``ydb.topic.reader.partition_session.count``) through ``observe_gauge``. A backend that has no notion of asynchronous gauges can simply store the callbacks (or ignore them); the SDK never pushes those values, so nothing is lost elsewhere. ``disable_metrics()`` reverts event recording to the no-op default and deactivates the -current backend's gauge callbacks. Trackers belonging to already instrumented pools -remain available so a replacement backend starts with an accurate snapshot; pool and -session lifecycle trackers that started on the no-op path remain no-ops. +current backend's gauge callbacks. Trackers belonging to already instrumented pools and +topic Readers remain available so a replacement backend starts with an accurate +snapshot; lifecycle trackers that started on the no-op path remain no-ops. diff --git a/docs/opentelemetry.rst b/docs/opentelemetry.rst index 0dc8d6dca..edd71e5dd 100644 --- a/docs/opentelemetry.rst +++ b/docs/opentelemetry.rst @@ -8,12 +8,13 @@ driver initialization, and retries — into OpenTelemetry spans, and propagates context to the YDB server through gRPC metadata using the `W3C Trace Context `_ standard. Enabling metrics records client-side OpenTelemetry instruments for the same operations, retries, and -query session pools. +query session pools, as well as receive, delivery, and buffer metrics for explicitly +named topic Readers. Like every backend, it has a **zero-cost no-op path when disabled**: operations, query -sessions, and pools share no-op instrumentation without metric allocations, timers, -locks, or state tracking. The SDK does not import ``opentelemetry`` until you call -``enable_tracing()`` or ``enable_metrics()``. +sessions, pools, and topic Readers share no-op instrumentation without metric +allocations, timers, locks, or state tracking. The SDK does not import +``opentelemetry`` until you call ``enable_tracing()`` or ``enable_metrics()``. Installation @@ -139,6 +140,99 @@ The full list of instruments and their attributes is catalogued on the :doc:`observability` page. +.. _opentelemetry-topic-reader-metrics: + +Topic Reader Metrics +-------------------- + +Enable metrics before creating the Reader and pass an explicit, non-empty +``reader_name``. The name identifies one logical Reader and remains the same when its +underlying stream reconnects: + +.. code-block:: python + + import asyncio + import uuid + + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import ( + ConsoleMetricExporter, + PeriodicExportingMetricReader, + ) + + import ydb + from ydb.opentelemetry import enable_metrics + + metric_reader = PeriodicExportingMetricReader( + ConsoleMetricExporter(), + export_interval_millis=1000, + ) + meter_provider = MeterProvider(metric_readers=[metric_reader]) + enable_metrics(meter_provider) + + + async def read_one_message(): + topic_path = f"/local/otel-reader-metrics-example-{uuid.uuid4().hex}" + consumer = "orders-consumer" + + async with ydb.aio.Driver( + endpoint="grpc://localhost:2136", + database="/local", + ) as driver: + await driver.wait(timeout=5) + + await driver.topic_client.create_topic(topic_path, consumers=[consumer]) + try: + async with driver.topic_client.writer(topic_path) as writer: + await writer.write("OpenTelemetry topic Reader metrics") + + async with driver.topic_client.reader( + topic_path, + consumer=consumer, + reader_name="payments-worker", + ) as reader: + await asyncio.wait_for(reader.wait_message(), timeout=10) + await asyncio.sleep(2) + message = await asyncio.wait_for(reader.receive_message(), timeout=10) + await reader.commit_with_ack(message) + await asyncio.sleep(2) + finally: + await driver.topic_client.drop_topic(topic_path) + + + try: + asyncio.run(read_one_message()) + finally: + meter_provider.shutdown() + +This produces three cumulative counters and three observable instruments: + +* ``ydb.topic.reader.received.messages`` counts received messages before decoding. It + includes ``endpoint``, ``database``, ``reader.name``, optional ``consumer``, and + ``topic`` attributes. +* ``ydb.topic.reader.received.bytes`` adds ``ReadResponse.bytes_size`` once for every + response received by the logical Reader. It has the same Reader attributes but no + ``topic``, because one response may contain batches from several topics. +* ``ydb.topic.reader.delivered.messages`` counts messages returned to the application + by ``receive_message``, ``receive_batch``, or ``receive_batch_with_tx``. It carries + the Reader attributes and ``topic``. +* ``ydb.topic.reader.local_buffer.messages`` reports the current number of messages + retained by the logical Reader, including messages waiting for decoding and decoded + messages waiting for the application. It has the Reader attributes but no ``topic``. +* ``ydb.topic.reader.local_buffer.message_age.max`` reports, in seconds, the age of the + oldest retained message. It uses a monotonic clock, becomes ``0`` when the buffer is + empty, and has the Reader attributes but no ``topic``. +* ``ydb.topic.reader.partition_session.count`` reports active partition sessions per + ``topic``. Configured topics report ``0`` before the first session starts. The metric + has the Reader attributes and ``topic``, but deliberately has no ``partition`` + attribute. + +If ``reader_name`` is omitted or empty, the Reader still works and receives its normal +process-local diagnostic name, but these six metrics are not published. The runnable +``examples/opentelemetry/otel_example.py`` creates a topic, writes one message, reads it +with an explicitly named Reader, and exports all six metrics through OTLP. + + Instrumented Spans and Attributes --------------------------------- @@ -226,9 +320,11 @@ Running the Examples -------------------- The runnable script is ``examples/opentelemetry/otel_example.py`` (bank table + concurrent -Serializable transactions and ``app_startup`` / ``example_tli`` application spans). **Start -Docker (YDB or the full stack) first**, then install and run on the host — see -``examples/opentelemetry/README.md`` for the full order of commands and environment variables. +Serializable transactions, ``app_startup`` / ``example_tli`` application spans, and an +explicitly named topic Reader that emits receive, delivery, and buffer metrics). **Start Docker +(YDB or the full stack) first**, then install and run on the host — see +``examples/opentelemetry/README.md`` for the full order of commands and environment +variables. **Full stack in one command** (YDB + OTLP + Tempo + Grafana; the ``otel-example`` service is built from ``examples/opentelemetry/Dockerfile`` and runs the script once): diff --git a/docs/topic.rst b/docs/topic.rst index 999d8733c..b49081ff9 100644 --- a/docs/topic.rst +++ b/docs/topic.rst @@ -383,7 +383,10 @@ Reader Parameters ``reader_name`` identifies a logical reader in server-side diagnostics and SDK logs. If it is omitted or empty, the SDK generates a process-local name in the form ``reader-N``. The name -remains unchanged when the reader reconnects. Explicit names do not have to be unique. +remains unchanged when the reader reconnects. Explicit names do not have to be unique. Topic +Reader metrics are published only when the application supplies a +non-empty ``reader_name``. See :ref:`topic-reader-receive-metrics` for their semantics +and attributes. ``buffer_size_bytes`` controls how many bytes the server is allowed to send before the client signals that it is ready for more. The server will not exceed this limit. diff --git a/examples/opentelemetry/README.md b/examples/opentelemetry/README.md index 33fd68991..7653486c0 100644 --- a/examples/opentelemetry/README.md +++ b/examples/opentelemetry/README.md @@ -2,7 +2,44 @@ Async demo in [`otel_example.py`](otel_example.py): OTLP export, `enable_tracing()`, `enable_metrics()`, `app_startup` and `example_tli` application spans, SDK client -metrics, bank table, Serializable transactions (TLI-style load). +metrics, bank table, Serializable transactions (TLI-style load), and an explicitly +named topic Reader that emits `ydb.topic.reader.received.messages`, +`ydb.topic.reader.received.bytes`, `ydb.topic.reader.delivered.messages`, +`ydb.topic.reader.local_buffer.messages`, +`ydb.topic.reader.local_buffer.message_age.max`, and +`ydb.topic.reader.partition_session.count`. + +## Topic Reader metrics in the example + +The demo enables metrics before creating the Reader and supplies +`reader_name="otel-example-reader"`. It creates a unique topic for the run, deletes only +that topic during cleanup, then writes and reads one message, producing: + +- `ydb.topic.reader.received.messages`: the number of messages received before + decoding, split by `topic`; +- `ydb.topic.reader.received.bytes`: the total `ReadResponse.bytes_size` received by + that logical Reader, without a `topic` label because one response can contain data + from several topics; +- `ydb.topic.reader.delivered.messages`: the number of messages returned by + `receive_message`, `receive_batch`, or `receive_batch_with_tx`, split by `topic`; +- `ydb.topic.reader.local_buffer.messages`: the current number of messages retained by + the logical Reader, including decoding and ready-to-deliver queues, without `topic`; +- `ydb.topic.reader.local_buffer.message_age.max`: the age in seconds of the oldest + retained message, or `0` when the buffer is empty, without `topic`; +- `ydb.topic.reader.partition_session.count`: the current number of active partition + sessions, split by `topic` and without a `partition` label; configured topics report + `0` before their first session starts. + +All six metrics also carry `endpoint`, `database`, and `reader.name`; `consumer` is +included for consumer-based reads. They remain attached to the same logical Reader +across stream reconnects. Use the rate of the cumulative counters to compare incoming +traffic with messages visible to the application, and read the observable instruments +as current values. + +The example keeps the message buffered, then keeps the empty Reader open, for +`OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS` seconds in each state. Its default is 7 seconds, +which is longer than this stack's 5-second Prometheus scrape interval, so both the +positive and zero gauge samples are observable. [`load_tank.py`](load_tank.py) runs a small step-like load profile for the metrics dashboard: @@ -51,9 +88,36 @@ Grafana is provisioned with the **YDB Python SDK Metrics** dashboard. It uses Prometheus queries for SDK metrics such as `db_client_operation_duration`, `ydb_client_operation_failed`, `ydb_query_session_count`, `ydb_query_session_pending_requests`, `ydb_query_session_create_time`, and -`ydb_client_retry_duration`. Use Grafana Explore for ad-hoc traces through Tempo +`ydb_client_retry_duration`. The example also publishes the topic Reader received and +delivered message counters by topic, the reader-level received-byte counter, and the +reader-level local-buffer count and oldest-age gauges. It also publishes the active +partition-session count by topic. Use Grafana Explore for ad-hoc traces through Tempo and metrics through Prometheus. +In Grafana Explore, select the Prometheus data source and query: + +```promql +ydb_topic_reader_local_buffer_messages{reader_name="otel-example-reader"} +``` + +The oldest buffered-message age is available as: + +```promql +ydb_topic_reader_local_buffer_message_age_max{reader_name="otel-example-reader"} +``` + +Active partition sessions are available as: + +```promql +ydb_topic_reader_partition_session_count{reader_name="otel-example-reader"} +``` + +The three counters use Prometheus' conventional `_total` suffix, for example: + +```promql +ydb_topic_reader_received_messages_total{reader_name="otel-example-reader"} +``` + The SDK configures explicit OpenTelemetry histogram bucket boundaries for its own duration and retry-attempt metrics. Duration values are recorded in seconds, with sub-millisecond and millisecond-scale buckets so Grafana percentiles show @@ -110,6 +174,7 @@ python examples/opentelemetry/load_tank.py | `YDB_DATABASE` | default `/local` | | `OTEL_EXPORTER_OTLP_ENDPOINT` | e.g. `http://otel-collector:4317` | | `OTEL_SERVICE_NAME` | service name attached to exported telemetry | +| `OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS` | seconds to expose each buffer-gauge state, default `7` | | `LOAD_TANK_TOTAL_TIME` | total load duration in seconds, default `6000` | | `LOAD_TANK_WORKERS` | number of concurrent workers, default `40` | | `LOAD_TANK_POOL_SIZE` | query session pool size, default `20` | diff --git a/examples/opentelemetry/compose-e2e.yaml b/examples/opentelemetry/compose-e2e.yaml index e0bf9a5e4..325d00c2a 100644 --- a/examples/opentelemetry/compose-e2e.yaml +++ b/examples/opentelemetry/compose-e2e.yaml @@ -84,6 +84,7 @@ services: YDB_DATABASE: /local OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 OTEL_SERVICE_NAME: ydb-otel-example + OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS: "7" depends_on: ydb: condition: service_healthy diff --git a/examples/opentelemetry/otel_example.py b/examples/opentelemetry/otel_example.py index 3bdd6a6b8..190046991 100644 --- a/examples/opentelemetry/otel_example.py +++ b/examples/opentelemetry/otel_example.py @@ -1,4 +1,4 @@ -"""OpenTelemetry + YDB demo: bank table and concurrent transactions (TLI-style workload). +"""OpenTelemetry + YDB demo: table operations, retries, and topic Reader metrics. Uses ``disable_discovery=True`` so a single ``grpc://…`` to local YDB (e.g. ``local-ydb`` in Docker from the host) is not replaced by internal discovery addresses. @@ -8,6 +8,7 @@ import asyncio import os +import uuid from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter from opentelemetry.sdk.metrics import MeterProvider @@ -44,10 +45,38 @@ async def _bank_read_update(tx) -> None: pass +async def _topic_reader_metrics_demo( + driver: ydb.aio.Driver, + database: str, + observation_seconds: float, +) -> None: + topic_path = f"{database.rstrip('/')}/otel-reader-metrics-{uuid.uuid4().hex}" + consumer = "otel-example-consumer" + + await driver.topic_client.create_topic(topic_path, consumers=[consumer]) + try: + async with driver.topic_client.writer(topic_path) as writer: + await writer.write("OpenTelemetry topic Reader metrics") + + async with driver.topic_client.reader( + topic_path, + consumer, + reader_name="otel-example-reader", + ) as reader: + await asyncio.wait_for(reader.wait_message(), timeout=10) + await asyncio.sleep(observation_seconds) + message = await asyncio.wait_for(reader.receive_message(), timeout=10) + await reader.commit_with_ack(message) + await asyncio.sleep(observation_seconds) + finally: + await driver.topic_client.drop_topic(topic_path) + + async def main() -> None: endpoint = _env("YDB_ENDPOINT", "grpc://localhost:2136") database = _env("YDB_DATABASE", "/local") otlp_endpoint = _env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + topic_buffer_observation_seconds = float(_env("OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS", "7")) resource = Resource(attributes={"service.name": _env("OTEL_SERVICE_NAME", "ydb-otel-example")}) tracer_provider = TracerProvider(resource=resource) @@ -96,6 +125,9 @@ async def concurrent_task(task_num: int) -> None: final_rows = await pool.execute_with_retries("SELECT amount FROM bank WHERE id = 1") amount = int(list(final_rows[0].rows)[0]["amount"]) print(f"Final amount (after serializable retries): {amount}") + + print("Emitting topic Reader metrics...") + await _topic_reader_metrics_demo(driver, database, topic_buffer_observation_seconds) print("Application will shut down in 15 seconds...") await asyncio.sleep(15) tracer_provider.shutdown() diff --git a/setup.py b/setup.py index a65864683..71e4a760f 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,6 @@ install_requires=requirements, # requirements.txt extras_require={ "yc": ["yandexcloud", ], - "opentelemetry": ["opentelemetry-api>=1.0.0"], + "opentelemetry": ["opentelemetry-api>=1.12.0"], } ) diff --git a/tests/observability/test_metrics.py b/tests/observability/test_metrics.py index 94c8ead64..aa4059232 100644 --- a/tests/observability/test_metrics.py +++ b/tests/observability/test_metrics.py @@ -1,7 +1,9 @@ import asyncio import inspect +import time from contextlib import contextmanager -from unittest.mock import AsyncMock, MagicMock +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, call import pytest from opentelemetry.metrics import Meter @@ -79,6 +81,62 @@ def _assert_closed_metric(reader, pool_name, reason): ) +def _topic_reader_batch(message_count, topic, partition_session_id=1, reader_stream_id=1): + return SimpleNamespace( + messages=[object()] * message_count, + _partition_session=SimpleNamespace( + topic_path=topic, + id=partition_session_id, + reader_stream_id=reader_stream_id, + ), + ) + + +def _topic_read_response(partition_messages, bytes_size=30): + import datetime + + from ydb._grpc.grpcwrapper.ydb_topic import Codec, StreamReadMessage + + now = datetime.datetime(2023, 2, 3, 14, 15) + return StreamReadMessage.ReadResponse( + partition_data=[ + StreamReadMessage.ReadResponse.PartitionData( + partition_session_id=partition_session_id, + batches=[ + StreamReadMessage.ReadResponse.Batch( + message_data=[ + StreamReadMessage.ReadResponse.MessageData( + offset=offset, + seq_no=offset + 1, + created_at=now, + data=("message-%d" % offset).encode(), + metadata_items={}, + uncompresed_size=10, + message_group_id="group", + ) + for offset in range(message_count) + ], + producer_id="producer", + write_session_meta={}, + codec=Codec.CODEC_RAW, + written_at=now, + ) + ], + ) + for partition_session_id, message_count in partition_messages + ], + bytes_size=bytes_size, + ) + + +async def _decode_pending_topic_batches(stream): + while not stream._batches_to_decode.empty(): + batch = stream._batches_to_decode.get_nowait() + await stream._decode_batch_inplace(batch) + stream._add_batch_to_queue(batch) + stream._state_changed.set() + + def _histogram_boundaries_advisory_supported(): return "explicit_bucket_boundaries_advisory" in inspect.signature(Meter.create_histogram).parameters @@ -86,7 +144,7 @@ def _histogram_boundaries_advisory_supported(): def test_metrics_build_info_version(metrics_setup): from ydb.observability.metrics import _metrics_build_info_tokens - assert _metrics_build_info_tokens() == ["ydb-sdk-metrics/0.2.0"] + assert _metrics_build_info_tokens() == ["ydb-sdk-metrics/0.7.0"] def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): @@ -103,10 +161,17 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): QUERY_SESSION_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DURATION, + TOPIC_READER_DELIVERED_MESSAGES, + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, ATTEMPT_BUCKETS, DURATION_BUCKETS_SECONDS, RETRY_DURATION_BUCKETS_SECONDS, SessionMetrics, + TopicReaderMetrics, create_metrics_operation, record_query_session_count, record_query_session_create_time, @@ -116,7 +181,10 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): record_retry_metrics, ) - monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(side_effect=[1.0, 1.25])) + monkeypatch.setattr( + "ydb.observability.metrics.time.monotonic", + MagicMock(side_effect=[1.0, 1.25, 2.0, 2.5]), + ) with pytest.raises(issues.Unavailable): with create_metrics_operation("ExecuteQuery"): @@ -133,6 +201,15 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): record_query_session_pending_requests(1, "main") record_query_session_timeout("main") record_retry_metrics(0.75, 3) + reader_metrics = TopicReaderMetrics( + object(), + consumer_name="analytics", + reader_name="reader-test", + ) + reader_metrics.record_received_messages([_topic_reader_batch(2, "/Root/events")]) + reader_metrics.record_received_bytes(128) + reader_metrics.record_delivered_messages(2, "/Root/events") + reader_metrics.add_partition_session("/Root/events") metrics = _metrics_by_name(metrics_setup) @@ -148,6 +225,12 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): QUERY_SESSION_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DURATION, + TOPIC_READER_DELIVERED_MESSAGES, + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, } assert metrics[CLIENT_OPERATION_DURATION].unit == "s" assert metrics[CLIENT_OPERATION_FAILED].unit == "{command}" @@ -160,6 +243,14 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): assert metrics[QUERY_SESSION_TIMEOUTS].unit == "{connection}" assert metrics[RETRY_DURATION].unit == "s" assert metrics[RETRY_ATTEMPTS].unit == "{attempt}" + assert metrics[TOPIC_READER_DELIVERED_MESSAGES].unit == "{message}" + assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX].unit == "s" + assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGES].unit == "{message}" + assert metrics[TOPIC_READER_PARTITION_SESSION_COUNT].unit == "{session}" + assert metrics[TOPIC_READER_RECEIVED_MESSAGES].unit == "{message}" + assert metrics[TOPIC_READER_RECEIVED_BYTES].unit == "By" + assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX].data.__class__.__name__ == "Gauge" + assert metrics[TOPIC_READER_PARTITION_SESSION_COUNT].data.__class__.__name__ == "Sum" if _histogram_boundaries_advisory_supported(): assert ( _single_point_from_metrics(metrics, CLIENT_OPERATION_DURATION).explicit_bounds == DURATION_BUCKETS_SECONDS @@ -198,6 +289,9 @@ def create_up_down_counter(self, name, unit="", description=""): def create_observable_up_down_counter(self, name, callbacks=None, unit="", description=""): return FakeInstrument() + def create_observable_gauge(self, name, callbacks=None, unit="", description=""): + return FakeInstrument() + OtelMetricsProvider(FakeMeter()) @@ -224,11 +318,13 @@ def test_metrics_registry_is_noop_without_meter(monkeypatch): _NOOP_PROVIDER, _NOOP_QUERY_SESSION_POOL_METRICS, _NOOP_SESSION_METRICS, + _NOOP_TOPIC_READER_METRICS, QuerySessionPoolMetrics, SessionMetrics, create_query_session_pool_metrics, create_session_metrics, create_metrics_operation, + create_topic_reader_metrics, record_query_session_create_time, record_query_session_max, record_query_session_pending_requests, @@ -288,6 +384,29 @@ def test_metrics_registry_is_noop_without_meter(monkeypatch): noop_pool_metrics.on_released(noop_session) noop_pool_metrics.close() + def fail_on_iteration(): + raise AssertionError("noop topic reader metrics iterated over batches") + yield + + noop_reader_metrics = create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name="reader-test", + topics=fail_on_iteration(), + ) + assert noop_reader_metrics is _NOOP_TOPIC_READER_METRICS + + noop_reader_metrics.record_received_messages(fail_on_iteration()) + assert noop_reader_metrics.capture_received_at() is None + noop_reader_metrics.record_received_bytes(10) + noop_reader_metrics.record_delivered_messages(1, "/Root/events") + noop_reader_metrics.remove_local_buffer_messages(1, object()) + noop_reader_metrics.reset_local_buffer_messages() + noop_reader_metrics.add_partition_session("/Root/events") + noop_reader_metrics.remove_partition_session("/Root/events") + noop_reader_metrics.reset_partition_sessions() + noop_reader_metrics.close() + assert not monotonic.called assert not noop_add.called pool_metrics.close() @@ -883,6 +1002,55 @@ def test_reconfigure_metrics_quiesces_old_gauge_callbacks(self): first_provider.shutdown() second_provider.shutdown() + def test_topic_reader_gauges_keep_state_while_metrics_are_reconfigured(self, monkeypatch): + from ydb.observability.metrics import ( + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + create_topic_reader_metrics, + ) + from ydb.opentelemetry import disable_metrics, enable_metrics + + first_reader = InMemoryMetricReader() + first_provider = MeterProvider(metric_readers=[first_reader]) + second_reader = InMemoryMetricReader() + second_provider = MeterProvider(metric_readers=[second_reader]) + + disable_metrics() + enable_metrics(first_provider) + reader_metrics = create_topic_reader_metrics(object(), consumer_name=None, reader_name="reconfigured") + first_batch = _topic_reader_batch(2, "/Root/events") + reader_metrics.record_received_messages( + [first_batch], + received_at=10.0, + ) + reader_metrics.add_partition_session("/Root/events") + monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=30.0)) + try: + assert _single_point(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 2 + assert _single_point(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0 + assert _single_point(first_reader, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1 + + disable_metrics() + second_batch = _topic_reader_batch(3, "/Root/events") + reader_metrics.record_received_messages( + [second_batch], + received_at=20.0, + ) + reader_metrics.remove_local_buffer_messages(1, first_batch._partition_session) + reader_metrics.add_partition_session("/Root/events") + enable_metrics(second_provider) + + assert _points(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == [] + assert _single_point(second_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 4 + assert _single_point(second_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0 + assert _single_point(second_reader, TOPIC_READER_PARTITION_SESSION_COUNT).value == 2 + finally: + reader_metrics.close() + disable_metrics() + first_provider.shutdown() + second_provider.shutdown() + def test_gauge_callback_drops_snapshot_if_provider_changes_during_collection(self): from ydb.observability import disable_metrics, enable_metrics from ydb.observability import metrics @@ -1038,6 +1206,650 @@ def fake_import(name, *args, **kwargs): otel.disable_metrics() +def test_topic_reader_received_metrics_attributes(metrics_setup): + from tests.observability.conftest import FakeDriverConfig + from ydb.observability.metrics import ( + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, + create_topic_reader_metrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + reader_metrics = create_topic_reader_metrics( + FakeDriver(), + consumer_name="analytics", + reader_name="payments-worker", + ) + reader_metrics.record_received_messages([_topic_reader_batch(2, "/Root/events")]) + reader_metrics.record_received_bytes(128) + reader_metrics.record_received_messages([_topic_reader_batch(3, "/Root/events")]) + reader_metrics.record_received_bytes(256) + + expected_reader_attributes = { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + } + expected_message_attributes = dict(expected_reader_attributes, topic="/Root/events") + messages = _single_point(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES) + received_bytes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES) + assert (messages.value, messages.attributes) == (5, expected_message_attributes) + assert (received_bytes.value, received_bytes.attributes) == (384, expected_reader_attributes) + + +def test_topic_reader_received_messages_are_split_by_topic(metrics_setup): + from tests.observability.conftest import FakeDriverConfig + from ydb.observability.metrics import ( + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, + create_topic_reader_metrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + reader_metrics = create_topic_reader_metrics( + FakeDriver(), + consumer_name="analytics", + reader_name="payments-worker", + ) + reader_metrics.record_received_messages( + [ + _topic_reader_batch(2, "/Root/orders"), + _topic_reader_batch(3, "/Root/refunds"), + _topic_reader_batch(1, "/Root/orders"), + ] + ) + reader_metrics.record_received_bytes(128) + + message_points = _points(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES) + assert sorted((point.attributes["topic"], point.value) for point in message_points) == [ + ("/Root/orders", 3), + ("/Root/refunds", 3), + ] + + received_bytes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES) + assert received_bytes.value == 128 + assert "topic" not in received_bytes.attributes + + +def test_topic_reader_received_messages_are_aggregated_before_provider_calls(monkeypatch): + from ydb.observability import metrics + + provider = MagicMock() + monkeypatch.setattr(metrics, "_provider", provider) + reader_metrics = metrics.TopicReaderMetrics( + object(), + consumer_name="analytics", + reader_name="payments-worker", + ) + try: + reader_metrics.record_received_messages( + [ + _topic_reader_batch(2, "/Root/orders"), + _topic_reader_batch(3, "/Root/refunds"), + _topic_reader_batch(1, "/Root/orders"), + ] + ) + finally: + reader_metrics.close() + + base_attributes = { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + } + assert provider.add.call_args_list == [ + call( + metrics.TOPIC_READER_RECEIVED_MESSAGES, + 3, + dict(base_attributes, topic="/Root/orders"), + ), + call( + metrics.TOPIC_READER_RECEIVED_MESSAGES, + 3, + dict(base_attributes, topic="/Root/refunds"), + ), + ] + + +def test_topic_reader_delivered_messages_are_split_by_topic(metrics_setup): + from tests.observability.conftest import FakeDriverConfig + from ydb.observability.metrics import ( + TOPIC_READER_DELIVERED_MESSAGES, + create_topic_reader_metrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + reader_metrics = create_topic_reader_metrics( + FakeDriver(), + consumer_name="analytics", + reader_name="payments-worker", + ) + reader_metrics.record_delivered_messages(2, "/Root/orders") + reader_metrics.record_delivered_messages(3, "/Root/refunds") + reader_metrics.record_delivered_messages(1, "/Root/orders") + reader_metrics.record_delivered_messages(0, "/Root/ignored") + + points = _points(metrics_setup, TOPIC_READER_DELIVERED_MESSAGES) + values = {point.attributes["topic"]: (point.value, point.attributes) for point in points} + assert values == { + "/Root/orders": ( + 3, + { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/orders", + }, + ), + "/Root/refunds": ( + 3, + { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/refunds", + }, + ), + } + + +def test_topic_reader_local_buffer_messages_tracks_current_reader_buffer(metrics_setup): + from tests.observability.conftest import FakeDriverConfig + from ydb.observability.metrics import ( + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + create_topic_reader_metrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + reader_metrics = create_topic_reader_metrics( + FakeDriver(), + consumer_name="analytics", + reader_name="payments-worker", + ) + try: + orders_batch = _topic_reader_batch(2, "/Root/orders", partition_session_id=1) + refunds_batch = _topic_reader_batch(3, "/Root/refunds", partition_session_id=2) + reader_metrics.record_received_messages([orders_batch, refunds_batch]) + + point = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) + assert (point.value, point.attributes) == ( + 5, + { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + }, + ) + assert "topic" not in point.attributes + + reader_metrics.remove_local_buffer_messages(2, orders_batch._partition_session) + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 3 + + reader_metrics.reset_local_buffer_messages() + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0 + finally: + reader_metrics.close() + + assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == [] + + +def test_topic_reader_local_buffer_age_tracks_oldest_message_per_partition(metrics_setup, monkeypatch): + from ydb.observability import metrics + + reader_metrics = metrics.create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name="payments-worker", + ) + first_partition_early = _topic_reader_batch( + 1, + "/Root/events", + partition_session_id=1, + ) + second_partition = _topic_reader_batch( + 1, + "/Root/events", + partition_session_id=2, + ) + first_partition_late = _topic_reader_batch( + 1, + "/Root/events", + partition_session_id=1, + ) + + try: + assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 0 + + reader_metrics.record_received_messages([first_partition_early], received_at=10.0) + reader_metrics.record_received_messages([second_partition], received_at=20.0) + reader_metrics.record_received_messages([first_partition_late], received_at=30.0) + monkeypatch.setattr(metrics.time, "monotonic", MagicMock(return_value=40.0)) + + age = _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX) + assert (age.value, age.attributes) == ( + 30.0, + { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + }, + ) + + reader_metrics.remove_local_buffer_messages(1, second_partition._partition_session) + assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 30.0 + + reader_metrics.remove_local_buffer_messages(1, first_partition_early._partition_session) + assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 10.0 + + reader_metrics.remove_local_buffer_messages(1, first_partition_late._partition_session) + assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 0 + assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0 + finally: + reader_metrics.close() + + assert _points(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX) == [] + + +def test_topic_reader_partition_session_count_is_split_by_topic(metrics_setup): + from ydb.observability.metrics import ( + TOPIC_READER_PARTITION_SESSION_COUNT, + create_topic_reader_metrics, + ) + + reader_metrics = create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name="payments-worker", + topics=("/Root/orders", "/Root/refunds"), + ) + try: + assert { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) + } == { + "/Root/orders": 0, + "/Root/refunds": 0, + } + + reader_metrics.add_partition_session("/Root/orders") + reader_metrics.add_partition_session("/Root/orders") + reader_metrics.add_partition_session("/Root/refunds") + + points = _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) + assert {point.attributes["topic"]: (point.value, point.attributes) for point in points} == { + "/Root/orders": ( + 2, + { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/orders", + }, + ), + "/Root/refunds": ( + 1, + { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/refunds", + }, + ), + } + + reader_metrics.remove_partition_session("/Root/orders") + reader_metrics.remove_partition_session("/Root/refunds") + points = _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) + assert {point.attributes["topic"]: point.value for point in points} == { + "/Root/orders": 1, + "/Root/refunds": 0, + } + + reader_metrics.reset_partition_sessions() + assert { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) + } == { + "/Root/orders": 0, + "/Root/refunds": 0, + } + finally: + reader_metrics.close() + + assert _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) == [] + + +@pytest.mark.asyncio +async def test_topic_reader_metrics_follow_reader_lifecycle_across_topics(metrics_setup): + from unittest import mock + + from tests.observability.conftest import FakeDriverConfig + from ydb._topic_reader.datatypes import PartitionSession + from ydb._topic_reader.topic_reader import PublicReaderSettings + from ydb._topic_reader.topic_reader_asyncio import PublicAsyncIOReader, ReaderStream + from ydb.observability.metrics import ( + TOPIC_READER_DELIVERED_MESSAGES, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + settings = PublicReaderSettings( + consumer="analytics", + topic="/Root/orders", + reader_name="payments-worker", + ) + stream_ready = asyncio.Event() + stream_holder = [] + + async def create_stream(reader_reconnector_id, driver, stream_settings, *, reader_name, metrics): + stream = ReaderStream( + reader_reconnector_id, + stream_settings, + reader_name=reader_name, + metrics=metrics, + ) + for session_id, topic in [(1, "/Root/orders"), (2, "/Root/refunds")]: + stream._partition_sessions[session_id] = PartitionSession( + id=session_id, + state=PartitionSession.State.Active, + topic_path=topic, + partition_id=session_id, + committed_offset=0, + reader_reconnector_id=reader_reconnector_id, + reader_stream_id=stream._id, + ) + stream_holder.append(stream) + stream_ready.set() + return stream + + reader_attributes = { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + } + with mock.patch.object(ReaderStream, "create", create_stream): + reader = PublicAsyncIOReader(FakeDriver(), settings) + try: + await asyncio.wait_for(stream_ready.wait(), timeout=1) + stream = stream_holder[0] + assert ( + _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value, + _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).attributes, + ) == (0, reader_attributes) + + stream._on_read_response(_topic_read_response([(1, 2), (2, 1)])) + await _decode_pending_topic_batches(stream) + + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 3 + assert len(_points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES)) == 1 + assert _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES).value == 30 + + assert await reader.receive_message() is not None + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 2 + assert len((await reader.receive_batch(max_messages=1)).messages) == 1 + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 1 + assert len((await reader.receive_batch()).messages) == 1 + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0 + + received = { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES) + } + delivered = { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_DELIVERED_MESSAGES) + } + assert received == {"/Root/orders": 2, "/Root/refunds": 1} + assert delivered == received + finally: + await reader.close(flush=False) + + assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == [] + + +@pytest.mark.asyncio +async def test_topic_reader_gauges_reset_across_stream_reconnect(metrics_setup, monkeypatch): + from unittest import mock + + from tests.observability.conftest import FakeDriverConfig + from ydb._topic_reader.datatypes import PartitionSession + from ydb._topic_reader.topic_reader import PublicReaderSettings + from ydb._topic_reader.topic_reader_asyncio import PublicAsyncIOReader, ReaderStream + from ydb.observability.metrics import ( + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + TopicReaderMetrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig() + + settings = PublicReaderSettings( + consumer="analytics", + topic="/Root/orders", + reader_name="reconnecting-reader", + ) + first_stream_ready = asyncio.Event() + second_stream_create_started = asyncio.Event() + allow_second_stream = asyncio.Event() + second_stream_ready = asyncio.Event() + streams = [] + monkeypatch.setattr( + TopicReaderMetrics, + "capture_received_at", + lambda self: time.monotonic() - 5.0, + ) + + async def create_stream(reader_reconnector_id, driver, stream_settings, *, reader_name, metrics): + stream = ReaderStream( + reader_reconnector_id, + stream_settings, + reader_name=reader_name, + metrics=metrics, + ) + stream._partition_sessions[1] = PartitionSession( + id=1, + state=PartitionSession.State.Active, + topic_path="/Root/orders", + partition_id=1, + committed_offset=0, + reader_reconnector_id=reader_reconnector_id, + reader_stream_id=stream._id, + ) + streams.append(stream) + if len(streams) == 1: + metrics.add_partition_session("/Root/orders") + stream._on_read_response(_topic_read_response([(1, 2)], bytes_size=20)) + await _decode_pending_topic_batches(stream) + first_stream_ready.set() + else: + second_stream_create_started.set() + await allow_second_stream.wait() + metrics.add_partition_session("/Root/orders") + stream._on_read_response(_topic_read_response([(1, 1)], bytes_size=10)) + await _decode_pending_topic_batches(stream) + second_stream_ready.set() + return stream + + with mock.patch.object(ReaderStream, "create", create_stream): + reader = PublicAsyncIOReader(FakeDriver(), settings) + try: + await asyncio.wait_for(first_stream_ready.wait(), timeout=1) + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 2 + first_age = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value + assert 5.0 <= first_age < 10.0 + assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1 + + streams[0]._set_first_error(issues.Unavailable("reconnect")) + await asyncio.wait_for(second_stream_create_started.wait(), timeout=2) + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0 + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 0 + assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 0 + + allow_second_stream.set() + await asyncio.wait_for(second_stream_ready.wait(), timeout=1) + + async def wait_until_second_stream_is_installed(): + while reader._reconnector._stream_reader is not streams[1]: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_until_second_stream_is_installed(), timeout=1) + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 1 + second_age = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value + assert 5.0 <= second_age < 10.0 + assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1 + finally: + allow_second_stream.set() + await reader.close(flush=False) + + assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == [] + assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX) == [] + assert _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) == [] + + +def test_topic_reader_gauges_aggregate_duplicate_reader_attributes(metrics_setup, monkeypatch): + from ydb.observability.metrics import ( + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + create_topic_reader_metrics, + ) + + first = create_topic_reader_metrics(object(), consumer_name=None, reader_name="shared") + second = create_topic_reader_metrics(object(), consumer_name=None, reader_name="shared") + try: + first.record_received_messages([_topic_reader_batch(2, "/Root/events")], received_at=10.0) + second.record_received_messages([_topic_reader_batch(3, "/Root/events")], received_at=20.0) + first.add_partition_session("/Root/events") + first.add_partition_session("/Root/events") + second.add_partition_session("/Root/events") + monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=30.0)) + + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 5 + assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0 + assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 3 + finally: + first.close() + second.close() + + +def test_topic_reader_received_metrics_omit_consumer(metrics_setup): + from ydb.observability.metrics import ( + TOPIC_READER_DELIVERED_MESSAGES, + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, + create_topic_reader_metrics, + ) + + reader_metrics = create_topic_reader_metrics( + object(), + consumer_name=None, + reader_name="reader-42", + ) + try: + reader_metrics.record_received_messages([_topic_reader_batch(1, "/Root/events")]) + reader_metrics.record_received_bytes(10) + reader_metrics.record_delivered_messages(1, "/Root/events") + reader_metrics.add_partition_session("/Root/events") + + reader_attributes = { + "endpoint": "", + "database": "", + "reader.name": "reader-42", + } + message_attributes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES).attributes + bytes_attributes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES).attributes + delivered_attributes = _single_point(metrics_setup, TOPIC_READER_DELIVERED_MESSAGES).attributes + buffer_attributes = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).attributes + buffer_age_attributes = _single_point( + metrics_setup, + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + ).attributes + partition_session_attributes = _single_point( + metrics_setup, + TOPIC_READER_PARTITION_SESSION_COUNT, + ).attributes + assert message_attributes == dict(reader_attributes, topic="/Root/events") + assert bytes_attributes == reader_attributes + assert delivered_attributes == dict(reader_attributes, topic="/Root/events") + assert buffer_attributes == reader_attributes + assert buffer_age_attributes == reader_attributes + assert partition_session_attributes == dict(reader_attributes, topic="/Root/events") + finally: + reader_metrics.close() + + +@pytest.mark.parametrize("reader_name", [None, ""]) +def test_topic_reader_received_metrics_require_explicit_reader_name(metrics_setup, reader_name): + from ydb.observability.metrics import ( + TOPIC_READER_DELIVERED_MESSAGES, + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, + create_topic_reader_metrics, + ) + + reader_metrics = create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name=reader_name, + ) + reader_metrics.record_received_messages([_topic_reader_batch(1, "/Root/events")]) + reader_metrics.record_received_bytes(10) + reader_metrics.record_delivered_messages(1, "/Root/events") + reader_metrics.add_partition_session("/Root/events") + + metrics = _metrics_by_name(metrics_setup) + assert TOPIC_READER_RECEIVED_MESSAGES not in metrics + assert TOPIC_READER_RECEIVED_BYTES not in metrics + assert TOPIC_READER_DELIVERED_MESSAGES not in metrics + assert TOPIC_READER_LOCAL_BUFFER_MESSAGES not in metrics + assert TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX not in metrics + assert TOPIC_READER_PARTITION_SESSION_COUNT not in metrics + + class TestEndpointHelpers: @pytest.mark.parametrize( "endpoint,expected", diff --git a/ydb/_topic_reader/topic_reader_asyncio.py b/ydb/_topic_reader/topic_reader_asyncio.py index 11c56ef85..8d6371be8 100644 --- a/ydb/_topic_reader/topic_reader_asyncio.py +++ b/ydb/_topic_reader/topic_reader_asyncio.py @@ -35,6 +35,11 @@ import logging from ..query.base import TxEvent +from ..observability.metrics import ( + TopicReaderMetrics, + _NOOP_TOPIC_READER_METRICS, + create_topic_reader_metrics, +) if typing.TYPE_CHECKING: from ..query.transaction import BaseQueryTxContext @@ -251,6 +256,7 @@ class ReaderReconnector: _driver: Driver _background_tasks: Set[Task] _reader_name: str + _metrics: TopicReaderMetrics _log_prefix: str _state_changed: asyncio.Event @@ -269,6 +275,12 @@ def __init__( self._settings = settings self._driver = driver self._reader_name = settings.reader_name or "reader-%d" % self._id + self._metrics = create_topic_reader_metrics( + driver, + consumer_name=settings.consumer, + reader_name=settings.reader_name, + topics=settings.topic, + ) self._log_prefix = "topic reader reader_name=%r reader_id=%s" % (self._reader_name, self._id) self._loop = loop if loop is not None else asyncio.get_running_loop() self._background_tasks = set() @@ -294,6 +306,7 @@ async def _connection_loop(self): self._driver, self._settings, reader_name=self._reader_name, + metrics=self._metrics, ) logger.debug("%s connected stream_id=%s", self._log_prefix, self._stream_reader._id) attempt = 0 @@ -347,10 +360,16 @@ async def wait_message(self): def receive_batch_nowait(self, max_messages: Optional[int] = None, max_bytes: Optional[int] = None): if self._stream_reader is None: return None - return self._stream_reader.receive_batch_nowait( + batch = self._stream_reader.receive_batch_nowait( max_messages=max_messages, max_bytes=max_bytes, ) + if batch is not None: + self._metrics.record_delivered_messages( + len(batch.messages), + batch._partition_session.topic_path, + ) + return batch def receive_batch_with_tx_nowait( self, tx: "BaseQueryTxContext", max_messages: Optional[int] = None, max_bytes: Optional[int] = None @@ -371,10 +390,21 @@ def receive_batch_with_tx_nowait( tx._add_callback(TxEvent.AFTER_COMMIT, batch._update_partition_offsets, self._loop) + self._metrics.record_delivered_messages( + len(batch.messages), + batch._partition_session.topic_path, + ) + return batch def receive_message_nowait(self): - return self._stream_reader.receive_message_nowait() + message = self._stream_reader.receive_message_nowait() + if message is not None: + self._metrics.record_delivered_messages( + 1, + message._partition_session.topic_path, + ) + return message def _init_tx(self, tx: "BaseQueryTxContext"): tx_id = tx.tx_id @@ -499,15 +529,21 @@ async def close(self, flush: bool): # close this flushes pending commits; cancelling the loop first would let it close # the stream with flush=False instead and skip the flush. self._closed = True - if self._stream_reader: - await self._stream_reader.close(flush) - # Wake any pending wait_message() waiter (e.g. a concurrent receive) so it doesn't - # hang if the loop was reconnecting when close() cancelled it. - self._set_first_error(TopicReaderStreamClosedError()) - for task in self._background_tasks: - task.cancel() - - await asyncio.wait(self._background_tasks) + try: + if self._stream_reader: + await self._stream_reader.close(flush) + finally: + try: + # Wake any pending wait_message() waiter (e.g. a concurrent receive) so it doesn't + # hang if the loop was reconnecting when close() cancelled it. + self._set_first_error(TopicReaderStreamClosedError()) + for task in self._background_tasks: + task.cancel() + + if self._background_tasks: + await asyncio.wait(self._background_tasks) + finally: + self._metrics.close() async def flush(self): if self._stream_reader: @@ -535,6 +571,7 @@ class ReaderStream: _id: int _reader_reconnector_id: int _reader_name: str + _metrics: TopicReaderMetrics _session_id: str _stream: Optional[IGrpcWrapperAsyncIO] _started: bool @@ -568,11 +605,13 @@ def __init__( get_token_function: Optional[Callable[[], str]] = None, *, reader_name: str, + metrics: TopicReaderMetrics = _NOOP_TOPIC_READER_METRICS, ): self._loop = asyncio.get_running_loop() self._id = ReaderStream._static_id_counter.inc_and_get() self._reader_reconnector_id = reader_reconnector_id self._reader_name = reader_name + self._metrics = metrics self._session_id = "not initialized" self._log_prefix = "topic reader reader_name=%r reader_id=%s stream_id=%s session_id=%s" % ( self._reader_name, @@ -614,6 +653,7 @@ async def create( settings: topic_reader.PublicReaderSettings, *, reader_name: str, + metrics: TopicReaderMetrics = _NOOP_TOPIC_READER_METRICS, ) -> "ReaderStream": stream = GrpcWrapperAsyncIO(StreamReadMessage.FromServer.from_proto) reader = None @@ -626,6 +666,7 @@ async def create( settings, get_token_function=creds.get_auth_token if creds else None, reader_name=reader_name, + metrics=metrics, ) await reader._start(stream, settings._init_message(reader_name=reader_name)) except BaseException: @@ -726,6 +767,10 @@ def receive_batch_nowait(self, max_messages: Optional[int] = None, max_bytes: Op part_sess_id, batch = self._get_first_batch() cutted_batch = batch._pop_batch(max_messages=max_messages, max_bytes=max_bytes) + self._metrics.remove_local_buffer_messages( + len(cutted_batch.messages), + cutted_batch._partition_session, + ) if not batch.empty(): self._return_batch_to_queue(part_sess_id, batch) @@ -745,6 +790,7 @@ def receive_message_nowait(self): part_sess_id, batch = self._get_first_batch() message, msgs_left = batch._pop() + self._metrics.remove_local_buffer_messages(1, message._partition_session) if not msgs_left: self._buffer_release_bytes(batch._bytes_size) @@ -896,7 +942,7 @@ async def _on_start_partition_session(self, message: StreamReadMessage.StartPart "Double start partition session: %s" % message.partition_session.partition_session_id ) - self._partition_sessions[message.partition_session.partition_session_id] = datatypes.PartitionSession( + partition_session = datatypes.PartitionSession( id=message.partition_session.partition_session_id, state=datatypes.PartitionSession.State.Active, topic_path=message.partition_session.path, @@ -905,6 +951,8 @@ async def _on_start_partition_session(self, message: StreamReadMessage.StartPart reader_reconnector_id=self._reader_reconnector_id, reader_stream_id=self._id, ) + self._partition_sessions[partition_session.id] = partition_session + self._metrics.add_partition_session(partition_session.topic_path) read_offset = None @@ -938,6 +986,7 @@ def _on_partition_session_stop(self, message: StreamReadMessage.StopPartitionSes partition = self._partition_sessions.pop(message.partition_session_id) partition.close() + self._metrics.remove_partition_session(partition.topic_path) if message.graceful and self._stream is not None: self._stream.write( @@ -959,12 +1008,16 @@ def _on_end_partition_session(self, message: StreamReadMessage.EndPartitionSessi self._partition_sessions[message.partition_session_id].end() def _on_read_response(self, message: StreamReadMessage.ReadResponse): + received_at = self._metrics.capture_received_at() self._buffer_consume_bytes(message.bytes_size) + self._metrics.record_received_bytes(message.bytes_size) batches = self._read_response_to_batches(message) for batch in batches: self._batches_to_decode.put_nowait(batch) + self._metrics.record_received_messages(batches, received_at=received_at) + def _on_commit_response(self, message: StreamReadMessage.CommitOffsetResponse): for partition_offset in message.partitions_committed_offsets: if partition_offset.partition_session_id not in self._partition_sessions: @@ -1109,22 +1162,28 @@ async def close(self, flush: bool): self._closed = True logger.debug("%s close", self._log_prefix) - if flush: - await self.flush() - - self._set_first_error(TopicReaderStreamClosedError()) - self._state_changed.set() - if self._stream is not None: - self._stream.close() - - for session in self._partition_sessions.values(): - session.close() - self._partition_sessions.clear() + try: + if flush: + await self.flush() + finally: + try: + self._set_first_error(TopicReaderStreamClosedError()) + self._state_changed.set() + if self._stream is not None: + self._stream.close() - for task in self._background_tasks: - task.cancel() + for session in self._partition_sessions.values(): + session.close() + self._partition_sessions.clear() - if self._background_tasks: - await asyncio.wait(self._background_tasks) + for task in self._background_tasks: + task.cancel() + if self._background_tasks: + await asyncio.wait(self._background_tasks) + finally: + try: + self._metrics.reset_local_buffer_messages() + finally: + self._metrics.reset_partition_sessions() logger.debug("%s was closed", self._log_prefix) diff --git a/ydb/_topic_reader/topic_reader_asyncio_test.py b/ydb/_topic_reader/topic_reader_asyncio_test.py index 989d13181..7de4e5db3 100644 --- a/ydb/_topic_reader/topic_reader_asyncio_test.py +++ b/ydb/_topic_reader/topic_reader_asyncio_test.py @@ -37,6 +37,7 @@ from .._grpc.common.protos import ydb_status_codes_pb2 from .._constants import DEFAULT_INITIAL_RESPONSE_TIMEOUT +from ..observability.metrics import TopicReaderMetrics @pytest.fixture(autouse=True) @@ -311,6 +312,147 @@ def batch_size(): else: await wait_condition(lambda: batch_size() > initial_batch_size) + async def test_received_metrics_are_recorded_before_decoding(self, stream, default_reader_settings): + metrics = mock.Mock(spec=TopicReaderMetrics) + metrics.capture_received_at.return_value = 12.5 + reader = await self.get_started_reader( + stream, + default_reader_settings, + reader_name="payments-worker", + metrics=metrics, + ) + partition_session = datatypes.PartitionSession( + id=self.partition_session_id, + topic_path=default_reader_settings.topic, + partition_id=4, + state=datatypes.PartitionSession.State.Active, + committed_offset=self.partition_session_committed_offset, + reader_reconnector_id=self.default_reader_reconnector_id, + reader_stream_id=reader._id, + ) + reader._partition_sessions[partition_session.id] = partition_session + + def assert_bytes_recorded_on_receipt(bytes_size): + assert reader._batches_to_decode.qsize() == 0 + assert partition_session.id not in reader._message_batches + assert bytes_size == 1 + + def assert_accepted_for_decoding(batches, *, received_at): + assert reader._batches_to_decode.qsize() == 1 + assert partition_session.id not in reader._message_batches + assert len(batches) == 1 + assert len(batches[0].messages) == 2 + assert batches[0]._partition_session.topic_path == default_reader_settings.topic + assert received_at == 12.5 + + metrics.record_received_bytes.side_effect = assert_bytes_recorded_on_receipt + metrics.record_received_messages.side_effect = assert_accepted_for_decoding + + try: + await self.send_batch( + reader, + [ + self.create_message(partition_session, 1, 1), + self.create_message(partition_session, 2, 1), + ], + ) + metrics.record_received_messages.assert_called_once() + metrics.record_received_bytes.assert_called_once_with(1) + + first = reader.receive_batch_nowait(max_messages=1) + second = reader.receive_batch_nowait(max_messages=1) + assert len(first.messages) == 1 + assert len(second.messages) == 1 + assert metrics.remove_local_buffer_messages.call_args_list == [ + mock.call(1, partition_session), + mock.call(1, partition_session), + ] + finally: + await reader.close(False) + + metrics.reset_local_buffer_messages.assert_called_once_with() + metrics.reset_partition_sessions.assert_called_once_with() + + async def test_received_bytes_are_recorded_before_response_conversion(self, default_reader_settings): + metrics = mock.Mock(spec=TopicReaderMetrics) + reader = ReaderStream( + self.default_reader_reconnector_id, + default_reader_settings, + reader_name="payments-worker", + metrics=metrics, + ) + response = StreamReadMessage.ReadResponse(bytes_size=5, partition_data=[]) + + with mock.patch.object( + reader, + "_read_response_to_batches", + side_effect=RuntimeError("invalid response"), + ): + with pytest.raises(RuntimeError, match="invalid response"): + reader._on_read_response(response) + + metrics.record_received_bytes.assert_called_once_with(5) + metrics.record_received_messages.assert_not_called() + + async def test_received_bytes_are_recorded_once_for_multitopic_response(self, default_reader_settings): + metrics = mock.Mock(spec=TopicReaderMetrics) + reader = ReaderStream( + self.default_reader_reconnector_id, + default_reader_settings, + reader_name="payments-worker", + metrics=metrics, + ) + topics = ("/Root/a", "/Root/b") + for partition_session_id, topic in enumerate(topics, start=1): + reader._partition_sessions[partition_session_id] = datatypes.PartitionSession( + id=partition_session_id, + topic_path=topic, + partition_id=partition_session_id, + state=datatypes.PartitionSession.State.Active, + committed_offset=0, + reader_reconnector_id=self.default_reader_reconnector_id, + reader_stream_id=reader._id, + ) + + reader._on_read_response( + StreamReadMessage.ReadResponse( + bytes_size=5, + partition_data=[ + StreamReadMessage.ReadResponse.PartitionData( + partition_session_id=partition_session_id, + batches=[ + StreamReadMessage.ReadResponse.Batch( + message_data=[ + StreamReadMessage.ReadResponse.MessageData( + offset=0, + seq_no=0, + created_at=datetime.datetime(2023, 2, 3, 14, 15), + data=b"", + uncompresed_size=0, + message_group_id="", + metadata_items={}, + ) + ], + producer_id="", + write_session_meta={}, + codec=Codec.CODEC_RAW, + written_at=datetime.datetime(2023, 2, 3, 14, 15), + ) + ], + ) + for partition_session_id in reader._partition_sessions + ], + ) + ) + + metrics.record_received_messages.assert_called_once() + batches = metrics.record_received_messages.call_args.args[0] + assert [(len(batch.messages), batch._partition_session.topic_path) for batch in batches] == [ + (1, "/Root/a"), + (1, "/Root/b"), + ] + metrics.record_received_bytes.assert_called_once_with(5) + async def test_unknown_error(self, stream, stream_reader_finish_with_error): class TestError(Exception): pass @@ -425,6 +567,29 @@ async def test_close_ack_waiters_when_close_stream_reader( with pytest.raises(topic_reader_asyncio.PublicTopicReaderPartitionExpiredError): waiter.future.result() + @pytest.mark.parametrize( + "close_error", + [RuntimeError("flush failed"), asyncio.CancelledError()], + ids=["error", "cancelled"], + ) + async def test_close_resets_local_buffer_after_flush_failure(self, default_reader_settings, close_error): + metrics = mock.Mock(spec=TopicReaderMetrics) + reader = ReaderStream( + self.default_reader_reconnector_id, + default_reader_settings, + reader_name="payments-worker", + metrics=metrics, + ) + reader.flush = mock.AsyncMock(side_effect=close_error) + + with pytest.raises(type(close_error)): + await reader.close(flush=True) + + assert reader._closed + assert all(task.done() for task in reader._background_tasks) + metrics.reset_local_buffer_messages.assert_called_once_with() + metrics.reset_partition_sessions.assert_called_once_with() + async def test_flush(self, stream, stream_reader_started: ReaderStream, partition_session): offset = self.partition_session_committed_offset + 1 waiter = partition_session.add_waiter(offset) @@ -677,6 +842,9 @@ async def test_start_partition( default_reader_settings, partition_session, ): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + def session_count(): return len(stream_reader._partition_sessions) @@ -723,8 +891,12 @@ def session_count(): reader_reconnector_id=self.default_reader_reconnector_id, reader_stream_id=stream_reader._id, ) + metrics.add_partition_session.assert_called_once_with(test_topic_path) async def test_partition_stop_force(self, stream, stream_reader, partition_session): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + def session_count(): return len(stream_reader._partition_sessions) @@ -747,8 +919,12 @@ def session_count(): assert session_count() == initial_session_count - 1 assert partition_session.id not in stream_reader._partition_sessions + metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path) async def test_partition_stop_graceful(self, stream, stream_reader, partition_session): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + def session_count(): return len(stream_reader._partition_sessions) @@ -771,6 +947,7 @@ def session_count(): assert resp.client_message == StreamReadMessage.StopPartitionSessionResponse( partition_session_id=partition_session.id ) + metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path) stream.from_server.put_nowait( StreamReadMessage.FromServer( @@ -788,6 +965,9 @@ def session_count(): stream.from_client.get_nowait() async def test_end_partition_session(self, stream, stream_reader, partition_session): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + def session_count(): return len(stream_reader._partition_sessions) @@ -811,6 +991,7 @@ def session_count(): assert session_count() == initial_session_count assert partition_session.id in stream_reader._partition_sessions assert partition_session.ended + metrics.remove_partition_session.assert_not_called() @pytest.mark.parametrize( "graceful", @@ -1474,6 +1655,31 @@ async def test_receive_partial_batch_releases_only_cut_bytes(self, stream_reader assert len(remainder.messages) == 2 assert remainder._bytes_size == 60 # remainder stays buffered + async def test_local_buffer_is_updated_before_credit_release_failure(self, stream_reader, monkeypatch): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + partition_session = stub_partition_session() + stream_reader._message_batches = OrderedDict( + { + 0: PublicBatch( + messages=[stub_message(1), stub_message(2)], + _partition_session=partition_session, + _bytes_size=20, + _codec=Codec.CODEC_RAW, + ) + } + ) + monkeypatch.setattr( + stream_reader, + "_buffer_release_bytes", + mock.Mock(side_effect=RuntimeError("credit release failed")), + ) + + with pytest.raises(RuntimeError, match="credit release failed"): + stream_reader.receive_batch_nowait(max_messages=1) + + metrics.remove_local_buffer_messages.assert_called_once_with(1, partition_session) + async def test_receive_batch_nowait(self, stream, stream_reader, partition_session): assert stream_reader.receive_batch_nowait() is None @@ -1502,6 +1708,8 @@ async def test_receive_batch_nowait(self, stream, stream_reader, partition_sessi async def test_receive_message_nowait(self, stream, stream_reader, partition_session): assert stream_reader.receive_batch_nowait() is None + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics initial_buffer_size = stream_reader._buffer_size_bytes @@ -1526,6 +1734,7 @@ async def test_receive_message_nowait(self, stream, stream_reader, partition_ses mess = stream_reader.receive_message_nowait() assert mess.seqno == expected_seqno + assert metrics.remove_local_buffer_messages.call_args_list == [mock.call(1, partition_session)] * 3 assert stream_reader._buffer_size_bytes == initial_buffer_size - 2 * self.default_batch_size async def test_update_token(self, stream): @@ -1631,9 +1840,12 @@ async def timeout_receive(timeout=None): @pytest.mark.asyncio class TestReaderReconnector: - @pytest.mark.parametrize("configured_reader_name", [None, ""]) + @pytest.mark.parametrize("configured_reader_name", [None, "", "payments-worker"]) async def test_reconnect_on_repeatable_error(self, monkeypatch, configured_reader_name): test_error = issues.Overloaded("test error") + metrics = mock.Mock(spec=TopicReaderMetrics) + create_metrics = mock.Mock(return_value=metrics) + monkeypatch.setattr(topic_reader_asyncio, "create_topic_reader_metrics", create_metrics) async def wait_error(): raise test_error @@ -1664,9 +1876,11 @@ async def stream_create( settings: PublicReaderSettings, *, reader_name: str, + metrics: TopicReaderMetrics, ): nonlocal stream_index reader_names.append(reader_name) + reader_metrics.append(metrics) stream_index += 1 if stream_index == 1: return reader_stream_mock_with_error @@ -1676,6 +1890,7 @@ async def stream_create( raise Exception("unexpected create stream") reader_names = [] + reader_metrics = [] with mock.patch.object(ReaderStream, "create", stream_create): reconnector = ReaderReconnector( mock.Mock(), @@ -1684,10 +1899,131 @@ async def stream_create( await wait_for_fast(reconnector.wait_message()) assert reader_names == [reconnector._reader_name, reconnector._reader_name] - assert reconnector._reader_name == "reader-%d" % reconnector._id + assert reader_metrics == [metrics, metrics] + assert reconnector._metrics is metrics + expected_reader_name = configured_reader_name or "reader-%d" % reconnector._id + assert reconnector._reader_name == expected_reader_name + create_metrics.assert_called_once_with( + reconnector._driver, + consumer_name="", + reader_name=configured_reader_name, + topics=reconnector._settings.topic, + ) reader_stream_mock_with_error.wait_error.assert_any_await() reader_stream_mock_with_error.wait_messages.assert_any_await() await reconnector.close(flush=False) + metrics.close.assert_called_once_with() + + @pytest.mark.parametrize( + "close_error", + [RuntimeError("stream close failed"), asyncio.CancelledError()], + ids=["error", "cancelled"], + ) + async def test_close_unregisters_metrics_after_stream_close_failure(self, close_error): + stream = mock.Mock(spec=ReaderStream) + stream.close = mock.AsyncMock(side_effect=close_error) + metrics = mock.Mock(spec=TopicReaderMetrics) + reconnector = object.__new__(ReaderReconnector) + reconnector._log_prefix = "test-reader" + reconnector._closed = False + reconnector._stream_reader = stream + reconnector._first_error = asyncio.get_running_loop().create_future() + reconnector._state_changed = asyncio.Event() + reconnector._background_tasks = set() + reconnector._metrics = metrics + + with pytest.raises(type(close_error)): + await reconnector.close(flush=False) + + assert reconnector._closed + metrics.close.assert_called_once_with() + + async def test_receive_batch_records_only_returned_messages(self): + partition_session = stub_partition_session() + partition_session.topic_path = "/Root/orders" + batch = PublicBatch( + messages=[stub_message(1), stub_message(2)], + _partition_session=partition_session, + _bytes_size=20, + _codec=Codec.CODEC_RAW, + ) + stream = mock.Mock(spec=ReaderStream) + stream.receive_batch_nowait.side_effect = [batch, None] + metrics = mock.Mock(spec=TopicReaderMetrics) + reconnector = object.__new__(ReaderReconnector) + reconnector._stream_reader = stream + reconnector._metrics = metrics + + assert reconnector.receive_batch_nowait(max_messages=2, max_bytes=100) is batch + assert reconnector.receive_batch_nowait() is None + + metrics.record_delivered_messages.assert_called_once_with(2, "/Root/orders") + + async def test_receive_message_records_only_returned_message(self): + message = stub_message(1) + message._partition_session.topic_path = "/Root/orders" + stream = mock.Mock(spec=ReaderStream) + stream.receive_message_nowait.side_effect = [message, None] + metrics = mock.Mock(spec=TopicReaderMetrics) + reconnector = object.__new__(ReaderReconnector) + reconnector._stream_reader = stream + reconnector._metrics = metrics + + assert reconnector.receive_message_nowait() is message + assert reconnector.receive_message_nowait() is None + + metrics.record_delivered_messages.assert_called_once_with(1, "/Root/orders") + + async def test_receive_batch_with_tx_records_after_successful_delivery(self): + partition_session = stub_partition_session() + partition_session.topic_path = "/Root/orders" + batch = PublicBatch( + messages=[stub_message(1), stub_message(2), stub_message(3)], + _partition_session=partition_session, + _bytes_size=30, + _codec=Codec.CODEC_RAW, + ) + stream = mock.Mock(spec=ReaderStream) + stream.receive_batch_nowait.return_value = batch + metrics = mock.Mock(spec=TopicReaderMetrics) + tx = mock.Mock() + tx.tx_id = "tx-1" + reconnector = object.__new__(ReaderReconnector) + reconnector._stream_reader = stream + reconnector._metrics = metrics + reconnector._tx_to_batches_map = {} + reconnector._loop = asyncio.get_running_loop() + + result = reconnector.receive_batch_with_tx_nowait(tx, max_messages=3) + + assert result is batch + assert reconnector._tx_to_batches_map["tx-1"] == [batch] + metrics.record_delivered_messages.assert_called_once_with(3, "/Root/orders") + + async def test_receive_batch_with_tx_does_not_record_failed_delivery(self): + partition_session = stub_partition_session() + batch = PublicBatch( + messages=[stub_message(1)], + _partition_session=partition_session, + _bytes_size=10, + _codec=Codec.CODEC_RAW, + ) + stream = mock.Mock(spec=ReaderStream) + stream.receive_batch_nowait.return_value = batch + metrics = mock.Mock(spec=TopicReaderMetrics) + tx = mock.Mock() + tx.tx_id = "tx-1" + tx._add_callback.side_effect = [None, None, None, RuntimeError("callback failed")] + reconnector = object.__new__(ReaderReconnector) + reconnector._stream_reader = stream + reconnector._metrics = metrics + reconnector._tx_to_batches_map = {} + reconnector._loop = asyncio.get_running_loop() + + with pytest.raises(RuntimeError, match="callback failed"): + reconnector.receive_batch_with_tx_nowait(tx) + + metrics.record_delivered_messages.assert_not_called() async def test_close_during_reconnect_does_not_hang(self): # The connection loop must stop on reader.close() even while it is closing the old @@ -1720,7 +2056,7 @@ async def wait_forever(): create_calls = 0 - async def stream_create(reader_reconnector_id, driver, settings, *, reader_name): + async def stream_create(reader_reconnector_id, driver, settings, *, reader_name, metrics): nonlocal create_calls create_calls += 1 return stream1 if create_calls == 1 else stream2 diff --git a/ydb/observability/metrics.py b/ydb/observability/metrics.py index 6bc06015b..942568a21 100644 --- a/ydb/observability/metrics.py +++ b/ydb/observability/metrics.py @@ -2,11 +2,11 @@ The SDK records metrics only after :func:`ydb.observability.enable_metrics` installs a concrete :class:`MetricsProvider` (OpenTelemetry in :mod:`ydb.opentelemetry`, or any -custom one). Until then operations, query sessions and pools share no-op -instrumentation without metric allocations, timers, locks or state tracking. Metrics -stay independent from tracing, and the SDK never depends on ``opentelemetry`` being -importable. Enable metrics before creating query sessions or pools whose lifecycle -should be instrumented. +custom one). Until then operations, query sessions, pools and topic Readers share +no-op instrumentation without metric allocations, timers, locks or state tracking. +Metrics stay independent from tracing, and the SDK never depends on ``opentelemetry`` +being importable. Enable metrics before creating query sessions, pools or topic Readers +whose lifecycle should be instrumented. A provider handles three instrument kinds: @@ -23,7 +23,8 @@ import functools import inspect import weakref -from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Tuple, TypeVar, cast +from collections import deque +from typing import Any, Callable, Deque, Dict, Iterable, List, Optional, Protocol, Tuple, TypeVar, cast from ydb.observability._endpoint import split_endpoint @@ -38,8 +39,14 @@ QUERY_SESSION_MIN = "ydb.query.session.min" RETRY_ATTEMPTS = "ydb.client.retry.attempts" RETRY_DURATION = "ydb.client.retry.duration" +TOPIC_READER_RECEIVED_MESSAGES = "ydb.topic.reader.received.messages" +TOPIC_READER_RECEIVED_BYTES = "ydb.topic.reader.received.bytes" +TOPIC_READER_DELIVERED_MESSAGES = "ydb.topic.reader.delivered.messages" +TOPIC_READER_LOCAL_BUFFER_MESSAGES = "ydb.topic.reader.local_buffer.messages" +TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX = "ydb.topic.reader.local_buffer.message_age.max" +TOPIC_READER_PARTITION_SESSION_COUNT = "ydb.topic.reader.partition_session.count" -METRICS_SDK_BUILD_INFO = "ydb-sdk-metrics/0.2.0" +METRICS_SDK_BUILD_INFO = "ydb-sdk-metrics/0.7.0" DURATION_BUCKETS_SECONDS = ( 0.001, @@ -146,6 +153,7 @@ def observe_gauge(self, name, callback): _session_max_state: Dict[Tuple, int] = {} _live_session_metrics: "weakref.WeakSet[SessionMetrics]" = weakref.WeakSet() _live_pool_metrics: "weakref.WeakSet[QuerySessionPoolMetrics]" = weakref.WeakSet() +_live_topic_reader_metrics: "weakref.WeakSet[TopicReaderMetrics]" = weakref.WeakSet() def is_metrics_enabled() -> bool: @@ -182,10 +190,49 @@ def _observe_session_min() -> List[Tuple[float, Dict[str, Any]]]: return [(0, attrs) for _, attrs in _observe_session_max()] +def _observe_topic_reader_local_buffer_messages() -> List[Tuple[float, Dict[str, Any]]]: + with _gauge_lock: + values: Dict[Tuple, int] = {} + for reader_metrics in _live_topic_reader_metrics: + key = tuple(sorted(reader_metrics._base_attributes.items())) + values[key] = values.get(key, 0) + reader_metrics._local_buffer_messages + return [(value, dict(attrs)) for attrs, value in values.items()] + + +def _observe_topic_reader_local_buffer_message_age_max() -> List[Tuple[float, Dict[str, Any]]]: + now = time.monotonic() + with _gauge_lock: + values: Dict[Tuple, float] = {} + for reader_metrics in _live_topic_reader_metrics: + key = tuple(sorted(reader_metrics._base_attributes.items())) + oldest_received_at = min( + (cohorts[0][0] for cohorts in reader_metrics._local_buffer_received_at.values() if cohorts), + default=None, + ) + age = 0.0 if oldest_received_at is None else max(0.0, now - oldest_received_at) + values[key] = max(values.get(key, 0.0), age) + return [(value, dict(attrs)) for attrs, value in values.items()] + + +def _observe_topic_reader_partition_session_count() -> List[Tuple[float, Dict[str, Any]]]: + with _gauge_lock: + values: Dict[Tuple, int] = {} + for reader_metrics in _live_topic_reader_metrics: + for topic, count in reader_metrics._partition_session_counts.items(): + attributes = dict(reader_metrics._base_attributes) + attributes["topic"] = topic + key = tuple(sorted(attributes.items())) + values[key] = values.get(key, 0) + count + return [(value, dict(attrs)) for attrs, value in values.items()] + + _OBSERVABLE_GAUGES = ( (QUERY_SESSION_COUNT, _observe_session_count), (QUERY_SESSION_MAX, _observe_session_max), (QUERY_SESSION_MIN, _observe_session_min), + (TOPIC_READER_LOCAL_BUFFER_MESSAGES, _observe_topic_reader_local_buffer_messages), + (TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, _observe_topic_reader_local_buffer_message_age_max), + (TOPIC_READER_PARTITION_SESSION_COUNT, _observe_topic_reader_partition_session_count), ) @@ -266,7 +313,7 @@ def query_session_pool_name( def _metrics_build_info_tokens() -> List[str]: """Metrics' contribution to the ``x-ydb-sdk-build-info`` header. - Returns ``["ydb-sdk-metrics/0.2.0"]`` once a metrics backend is installed, + Returns ``["ydb-sdk-metrics/0.7.0"]`` once a metrics backend is installed, otherwise an empty list. Aggregated with other features by :func:`ydb.observability.sdk_build_info_tokens`. """ @@ -497,6 +544,213 @@ def __exit__(self, exc_type, exc_val, exc_tb): _NOOP_CM = _NoopContext() +class TopicReaderMetrics: + """Metric context shared by every stream of one logical topic reader.""" + + __slots__ = ( + "_base_attributes", + "_local_buffer_messages", + "_local_buffer_received_at", + "_partition_session_counts", + "_closed", + "__weakref__", + ) + + def __init__( + self, + driver, + consumer_name: Optional[str], + reader_name: str, + topics: Iterable[str] = (), + ) -> None: + driver_config = getattr(driver, "_driver_config", None) + self._base_attributes = _build_ydb_metrics_attrs(driver_config) + self._base_attributes["reader.name"] = reader_name + if consumer_name is not None: + self._base_attributes["consumer"] = consumer_name + self._local_buffer_messages = 0 + self._local_buffer_received_at: Dict[Tuple[int, int], Deque[Tuple[float, int]]] = {} + self._partition_session_counts: Dict[str, int] = dict.fromkeys(topics, 0) + self._closed = False + with _gauge_lock: + _live_topic_reader_metrics.add(self) + + @staticmethod + def _partition_session_key(partition_session: Any) -> Tuple[int, int]: + return partition_session.reader_stream_id, partition_session.id + + def capture_received_at(self) -> Optional[float]: + return time.monotonic() + + def record_received_messages(self, batches: Iterable[Any], received_at: Optional[float] = None) -> None: + messages_by_topic: Dict[str, int] = {} + messages_by_partition: Dict[Tuple[int, int], int] = {} + for batch in batches: + messages = len(batch.messages) + if messages > 0: + partition_session = batch._partition_session + topic = partition_session.topic_path + messages_by_topic[topic] = messages_by_topic.get(topic, 0) + messages + key = self._partition_session_key(partition_session) + messages_by_partition[key] = messages_by_partition.get(key, 0) + messages + + received_messages = sum(messages_by_partition.values()) + if received_messages > 0: + if received_at is None: + received_at = time.monotonic() + with _gauge_lock: + if not self._closed: + self._local_buffer_messages += received_messages + for key, messages in messages_by_partition.items(): + cohorts = self._local_buffer_received_at.setdefault(key, deque()) + if cohorts and cohorts[-1][0] == received_at: + _, previous_messages = cohorts.pop() + cohorts.append((received_at, previous_messages + messages)) + else: + cohorts.append((received_at, messages)) + + provider = _provider + if provider is _NOOP_PROVIDER: + return + + for topic, messages in messages_by_topic.items(): + attributes = dict(self._base_attributes) + attributes["topic"] = topic + provider.add(TOPIC_READER_RECEIVED_MESSAGES, messages, attributes) + + def record_received_bytes(self, bytes_size: int) -> None: + provider = _provider + if provider is _NOOP_PROVIDER: + return + + if bytes_size > 0: + provider.add(TOPIC_READER_RECEIVED_BYTES, bytes_size, dict(self._base_attributes)) + + def record_delivered_messages(self, message_count: int, topic: str) -> None: + provider = _provider + if provider is _NOOP_PROVIDER: + return + + if message_count > 0: + attributes = dict(self._base_attributes) + attributes["topic"] = topic + provider.add(TOPIC_READER_DELIVERED_MESSAGES, message_count, attributes) + + def remove_local_buffer_messages(self, message_count: int, partition_session: Any) -> None: + if message_count <= 0: + return + + key = self._partition_session_key(partition_session) + with _gauge_lock: + if self._closed: + return + + self._local_buffer_messages = max(0, self._local_buffer_messages - message_count) + cohorts = self._local_buffer_received_at.get(key) + messages_to_remove = message_count + while cohorts and messages_to_remove > 0: + received_at, messages = cohorts[0] + if messages <= messages_to_remove: + cohorts.popleft() + messages_to_remove -= messages + else: + cohorts[0] = (received_at, messages - messages_to_remove) + messages_to_remove = 0 + + if cohorts is not None and not cohorts: + self._local_buffer_received_at.pop(key, None) + if self._local_buffer_messages == 0: + self._local_buffer_received_at.clear() + + def reset_local_buffer_messages(self) -> None: + with _gauge_lock: + self._local_buffer_messages = 0 + self._local_buffer_received_at.clear() + + def add_partition_session(self, topic: str) -> None: + with _gauge_lock: + if self._closed: + return + self._partition_session_counts[topic] = self._partition_session_counts.get(topic, 0) + 1 + + def remove_partition_session(self, topic: str) -> None: + with _gauge_lock: + if self._closed or topic not in self._partition_session_counts: + return + self._partition_session_counts[topic] = max(0, self._partition_session_counts[topic] - 1) + + def reset_partition_sessions(self) -> None: + with _gauge_lock: + for topic in self._partition_session_counts: + self._partition_session_counts[topic] = 0 + + def close(self) -> None: + with _gauge_lock: + if self._closed: + return + self._closed = True + self._local_buffer_messages = 0 + self._local_buffer_received_at.clear() + self._partition_session_counts.clear() + _live_topic_reader_metrics.discard(self) + + +class _NoopTopicReaderMetrics(TopicReaderMetrics): + __slots__ = () + + def __init__(self) -> None: + pass + + def record_received_messages(self, batches: Iterable[Any], received_at: Optional[float] = None) -> None: + pass + + def capture_received_at(self) -> Optional[float]: + return None + + def record_received_bytes(self, bytes_size: int) -> None: + pass + + def record_delivered_messages(self, message_count: int, topic: str) -> None: + pass + + def remove_local_buffer_messages(self, message_count: int, partition_session: Any) -> None: + pass + + def reset_local_buffer_messages(self) -> None: + pass + + def add_partition_session(self, topic: str) -> None: + pass + + def remove_partition_session(self, topic: str) -> None: + pass + + def reset_partition_sessions(self) -> None: + pass + + def close(self) -> None: + pass + + +_NOOP_TOPIC_READER_METRICS = _NoopTopicReaderMetrics() + + +def create_topic_reader_metrics( + driver, + consumer_name: Optional[str], + reader_name: Optional[str], + topics: Any = (), +) -> TopicReaderMetrics: + """Create instrumentation only for explicitly named readers while metrics are active.""" + if not reader_name or not is_metrics_enabled(): + return _NOOP_TOPIC_READER_METRICS + + if isinstance(topics, str) or hasattr(topics, "path"): + topics = (topics,) + topic_paths = (topic if isinstance(topic, str) else topic.path for topic in topics) + return TopicReaderMetrics(driver, consumer_name, reader_name, topic_paths) + + class SessionMetrics: """Per-session query-session-count bookkeeping, kept out of the session's own code. diff --git a/ydb/opentelemetry/metrics_plugin.py b/ydb/opentelemetry/metrics_plugin.py index 3eed89df9..7fa31d6d9 100644 --- a/ydb/opentelemetry/metrics_plugin.py +++ b/ydb/opentelemetry/metrics_plugin.py @@ -30,6 +30,12 @@ QUERY_SESSION_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DURATION, + TOPIC_READER_DELIVERED_MESSAGES, + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_PARTITION_SESSION_COUNT, + TOPIC_READER_RECEIVED_BYTES, + TOPIC_READER_RECEIVED_MESSAGES, ATTEMPT_BUCKETS, DURATION_BUCKETS_SECONDS, GaugeCallback, @@ -45,7 +51,20 @@ QUERY_SESSION_COUNT: ("{connection}", "Number of open YDB query sessions."), QUERY_SESSION_MAX: ("{connection}", "Maximum configured number of YDB query sessions."), QUERY_SESSION_MIN: ("{connection}", "Minimum configured number of YDB query sessions."), + TOPIC_READER_LOCAL_BUFFER_MESSAGES: ( + "{message}", + "The number of messages currently buffered by the reader.", + ), + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX: ( + "s", + "Age of the oldest message currently buffered by the reader.", + ), + TOPIC_READER_PARTITION_SESSION_COUNT: ( + "{session}", + "The number of active partition sessions owned by the reader.", + ), } +_OBSERVABLE_GAUGE_INSTRUMENTS = frozenset({TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX}) class OtelMetricsProvider: @@ -114,6 +133,21 @@ def __init__(self, meter: Meter) -> None: unit="{request}", description="Number of requests waiting for a YDB query session.", ), + TOPIC_READER_DELIVERED_MESSAGES: meter.create_counter( + TOPIC_READER_DELIVERED_MESSAGES, + unit="{message}", + description="Number of messages delivered by the YDB topic reader to the application.", + ), + TOPIC_READER_RECEIVED_MESSAGES: meter.create_counter( + TOPIC_READER_RECEIVED_MESSAGES, + unit="{message}", + description="The number of messages accepted by the SDK for an active partition session.", + ), + TOPIC_READER_RECEIVED_BYTES: meter.create_counter( + TOPIC_READER_RECEIVED_BYTES, + unit="By", + description="The protocol bytes_size received in read responses.", + ), } def record(self, name: str, value: float, attributes: Optional[Dict[str, Any]] = None) -> None: @@ -132,7 +166,12 @@ def observe_gauge(self, name: str, callback: GaugeCallback) -> None: def _otel_callback(_: CallbackOptions): return [Observation(value, attributes=attrs) for value, attrs in callback()] - self._meter.create_observable_up_down_counter( + create_instrument = ( + self._meter.create_observable_gauge + if name in _OBSERVABLE_GAUGE_INSTRUMENTS + else self._meter.create_observable_up_down_counter + ) + create_instrument( name, callbacks=[_otel_callback], unit=unit, From bc4adbf872079744af21e229dbc110315e73406e Mon Sep 17 00:00:00 2001 From: nastya Date: Fri, 30 Oct 2026 16:59:46 +0300 Subject: [PATCH 3/5] Add all topic reader metrics --- .dockerignore | 3 +- CHANGELOG.md | 2 +- docs/observability.rst | 88 +- docs/opentelemetry.rst | 33 +- examples/opentelemetry/Dockerfile | 1 + examples/opentelemetry/README.md | 185 ++++- examples/opentelemetry/compose-e2e.yaml | 52 +- .../dashboards/ydb-python-sdk-metrics.json | 177 +++- examples/opentelemetry/load_tank.py | 383 +++++++-- examples/opentelemetry/otel_example.py | 2 +- tests/observability/test_metrics.py | 781 +++++++++++++++++- ydb/_topic_reader/datatypes.py | 13 +- ydb/_topic_reader/datatypes_test.py | 31 +- ydb/_topic_reader/topic_reader_asyncio.py | 134 ++- .../topic_reader_asyncio_test.py | 399 ++++++++- ydb/observability/metrics.py | 225 ++++- ydb/opentelemetry/metrics_plugin.py | 52 +- 17 files changed, 2439 insertions(+), 122 deletions(-) diff --git a/.dockerignore b/.dockerignore index 418b31ebe..e48815d6d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,5 @@ !setup.py !examples/opentelemetry/requirements.txt !examples/opentelemetry/otel_example.py -!examples/opentelemetry/load_tank.py \ No newline at end of file +!examples/opentelemetry/load_tank.py +!examples/opentelemetry/verify_metrics.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 97de48346..ede6f6be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -* Add `ydb.topic.reader.received.messages`, reader-level `ydb.topic.reader.received.bytes`, and `ydb.topic.reader.delivered.messages` counters plus `ydb.topic.reader.local_buffer.messages`, `ydb.topic.reader.local_buffer.message_age.max`, and `ydb.topic.reader.partition_session.count` observable metrics for explicitly named topic readers; metrics-enabled clients now advertise `ydb-sdk-metrics/0.7.0` in `x-ydb-sdk-build-info` +* Add `ydb.topic.reader.received.messages`, reader-level `ydb.topic.reader.received.bytes`, `ydb.topic.reader.delivered.messages`, `ydb.topic.reader.commit.queued`, `ydb.topic.reader.commit.acknowledged`, and `ydb.topic.reader.session.errors` counters, `ydb.topic.reader.compression.duration` and `ydb.topic.reader.message.processing.duration` histograms, plus `ydb.topic.reader.commit_offset.lag.max`, `ydb.topic.reader.credit_balance_bytes`, `ydb.topic.reader.local_buffer.messages`, `ydb.topic.reader.local_buffer.message_age.max`, and `ydb.topic.reader.partition_session.count` observable metrics for explicitly named topic readers; metrics-enabled clients now advertise `ydb-sdk-metrics/0.14.0` in `x-ydb-sdk-build-info` * Add optional topic reader names for identifying logical readers in server diagnostics and SDK logs ## 3.33.2 ## diff --git a/docs/observability.rst b/docs/observability.rst index f0b0bca92..e350ee1bc 100644 --- a/docs/observability.rst +++ b/docs/observability.rst @@ -362,6 +362,27 @@ adapter maps them to instruments on the ``"ydb.sdk"`` meter): * - ``ydb.topic.reader.delivered.messages`` - Counter (``{message}``) - Messages returned by the topic Reader to the application. + * - ``ydb.topic.reader.compression.duration`` + - Histogram (``s``) + - Time spent decompressing a message, excluding decoder-executor queueing. + * - ``ydb.topic.reader.message.processing.duration`` + - Histogram (``s``) + - Time from receipt until a message is ready for delivery. + * - ``ydb.topic.reader.commit.queued`` + - Counter (``{message}``) + - Messages in commit ranges accepted by the SDK. + * - ``ydb.topic.reader.commit.acknowledged`` + - Counter (``{message}``) + - Messages acknowledged by regular committed-offset advances or successful transactions. + * - ``ydb.topic.reader.commit_offset.lag.max`` + - ObservableGauge (``{message}``) + - Maximum gap between requested and acknowledged commit offsets across active partitions. + * - ``ydb.topic.reader.session.errors`` + - Counter (``{error}``) + - Reader session errors by retry decision, status code, and error type. + * - ``ydb.topic.reader.credit_balance_bytes`` + - ObservableGauge (``By``) + - Protocol credit granted to the server and not yet consumed by read responses. * - ``ydb.topic.reader.local_buffer.messages`` - ObservableUpDownCounter (``{message}``) - Current number of messages owned by the topic Reader and not yet returned to the application. @@ -429,14 +450,18 @@ Topic Reader Metrics Topic Reader metrics carry ``endpoint``, ``database``, and ``reader.name``. The ``consumer`` attribute is present only for consumer-based reads; the SDK omits it when reading without a consumer. ``ydb.topic.reader.received.messages``, -``ydb.topic.reader.delivered.messages``, and -``ydb.topic.reader.partition_session.count`` additionally carry ``topic`` because each -message and partition session has an unambiguous topic. The reader-level -``ydb.topic.reader.received.bytes``, ``ydb.topic.reader.local_buffer.messages``, and +``ydb.topic.reader.delivered.messages``, ``ydb.topic.reader.compression.duration``, +``ydb.topic.reader.message.processing.duration``, ``ydb.topic.reader.commit.queued``, +``ydb.topic.reader.commit.acknowledged``, ``ydb.topic.reader.commit_offset.lag.max``, +and ``ydb.topic.reader.partition_session.count`` additionally carry ``topic`` because +each message, commit offset, and partition session has an unambiguous topic. The reader-level +``ydb.topic.reader.received.bytes``, ``ydb.topic.reader.session.errors``, +``ydb.topic.reader.credit_balance_bytes``, +``ydb.topic.reader.local_buffer.messages``, and ``ydb.topic.reader.local_buffer.message_age.max`` metrics omit ``topic``: one ``ReadResponse`` can contain data from several topics, and the buffer gauges follow the -total retained by the logical Reader. Each response increments the byte counter once by -``ReadResponse.bytes_size``. +total retained by the logical Reader, while a session error belongs to the whole Reader. +Each response increments the byte counter once by ``ReadResponse.bytes_size``. Metrics are published only when the application passes a non-empty ``reader_name`` to ``TopicClient.reader``. The name and metric context belong to the logical Reader and remain stable across stream reconnects. The byte counter is updated as soon as the @@ -453,6 +478,48 @@ failures before the return point do not increment it. The local-buffer gauge inc when accepted batches enter the decoding queue and decreases when messages leave the internal buffer, including partial batch reads. It therefore includes both messages waiting for decoding and decoded messages waiting for the application. +For every non-RAW message, ``ydb.topic.reader.compression.duration`` records only time +inside the decoder callable; time waiting in the decoder executor is excluded. +``ydb.topic.reader.message.processing.duration`` starts when the ``ReadResponse`` is +received and ends when the decoded batch becomes ready for delivery, so it includes +response conversion and decoder queueing. Both histograms carry ``topic``, ``codec``, +and ``outcome=ok|error``; failed measurements also carry ``error.type``. A batch is the +delivery unit in the Python SDK, so every message in one batch receives the same +processing duration. If any decoder fails, the batch's processing measurements use +``outcome=error``. RAW messages require no decoding and do not create either histogram +measurement. Only completed measurements are recorded. +Each commit range accepted by the SDK increments ``ydb.topic.reader.commit.queued`` by +``range.end - range.start``. A regular commit is counted after its +``CommitOffsetRequest`` is queued on the active stream. A transactional commit is counted +after its ranges pass SDK validation and are submitted to the transaction update path. +Already acknowledged ranges, expired partition sessions, and regular commit requests +that cannot be queued are not counted. A later transport or transaction failure does not +undo this cumulative counter. +For a regular ``CommitOffsetResponse``, ``ydb.topic.reader.commit.acknowledged`` increments +by ``max(0, committed_offset - previous_committed_offset)``. This counts the monotonic +committed-offset advance once even when queued ranges overlap. Repeated and stale +responses add nothing. Transactional ranges are acknowledged by their sizes only after +the whole transaction commits successfully; a failed commit or rollback does not +increment the counter. +``ydb.topic.reader.commit_offset.lag.max`` reports the maximum, across active partition +sessions of each topic, of ``max(0, highest_requested_end_offset - acknowledged_offset)``. +It increases after a regular ``CommitOffsetRequest`` is queued and decreases after a +``CommitOffsetResponse`` advances the acknowledged offset. Transactional commit ranges +remain visible in the commit counters but do not contribute to this stream-ACK gauge. +A stale acknowledgement cannot increase it. Stopping a partition session removes that +partition's contribution; reconnecting replaces the old stream's state, and a topic +with no active partition sessions has no observation. +Each failed session or connection attempt increments ``ydb.topic.reader.session.errors`` +once after the retry policy classifies it. The ``retry_decision`` attribute is ``retry`` +when the Reader will reconnect and ``stop`` when the error terminates the Reader. +``status_code`` contains the YDB status name when available and otherwise the exception +class name. ``error.type`` is ``transport_error`` for transport failures, ``ydb_error`` +for other YDB errors, and the exception class name for other failures. Closing a Reader +normally does not increment the counter. +``credit_balance_bytes`` is set to the configured buffer size after the initial +``ReadRequest`` is sent, decreases by ``ReadResponse.bytes_size``, and increases only +when the Reader sends another ``ReadRequest`` after its release threshold is reached. +It is ``0`` while the logical Reader has no active stream, including reconnect gaps. ``local_buffer.message_age.max`` is computed with a monotonic clock from the receive time of the oldest retained message; it is ``0`` when the buffer is empty. Receipt cohorts are tracked internally per partition session so delivery from another partition @@ -463,8 +530,8 @@ even before its first partition session starts. Remaining messages and partition sessions are reset when a stream closes or reconnects, and closing the logical Reader removes its observations. Use rates of the cumulative counters to compare incoming and application-visible traffic; read the observable instruments as current values. For -matching Reader attributes, buffer counts and partition-session counts are summed while -the maximum buffer age is published. +matching Reader attributes, credit balances, buffer counts, and partition-session counts +are summed while the maximum buffer age is published. See :ref:`opentelemetry-topic-reader-metrics` for an OpenTelemetry setup example. A complete runnable version is available in ``examples/opentelemetry/otel_example.py``. @@ -510,8 +577,9 @@ subclass; just match the signatures: The instrument kind for each metric is implied by which method the SDK calls: names in the *Histogram* rows above arrive through ``record``, *Counter* / *UpDownCounter* names -through ``add``, and the six observable instruments (the three query-session gauges, -``ydb.topic.reader.local_buffer.messages``, +through ``add``, and the eight observable instruments (the three query-session gauges, +``ydb.topic.reader.commit_offset.lag.max``, +``ydb.topic.reader.credit_balance_bytes``, ``ydb.topic.reader.local_buffer.messages``, ``ydb.topic.reader.local_buffer.message_age.max``, and ``ydb.topic.reader.partition_session.count``) through ``observe_gauge``. A backend that has no notion of asynchronous gauges can simply store the callbacks (or ignore diff --git a/docs/opentelemetry.rst b/docs/opentelemetry.rst index edd71e5dd..d1bb679cd 100644 --- a/docs/opentelemetry.rst +++ b/docs/opentelemetry.rst @@ -183,7 +183,7 @@ underlying stream reconnects: await driver.topic_client.create_topic(topic_path, consumers=[consumer]) try: - async with driver.topic_client.writer(topic_path) as writer: + async with driver.topic_client.writer(topic_path, codec=ydb.TopicCodec.GZIP) as writer: await writer.write("OpenTelemetry topic Reader metrics") async with driver.topic_client.reader( @@ -205,7 +205,7 @@ underlying stream reconnects: finally: meter_provider.shutdown() -This produces three cumulative counters and three observable instruments: +This registers six cumulative counters, two histograms, and five observable instruments: * ``ydb.topic.reader.received.messages`` counts received messages before decoding. It includes ``endpoint``, ``database``, ``reader.name``, optional ``consumer``, and @@ -216,6 +216,31 @@ This produces three cumulative counters and three observable instruments: * ``ydb.topic.reader.delivered.messages`` counts messages returned to the application by ``receive_message``, ``receive_batch``, or ``receive_batch_with_tx``. It carries the Reader attributes and ``topic``. +* ``ydb.topic.reader.compression.duration`` records time spent inside the decoder for + each non-RAW message, excluding decoder-executor queueing. It uses seconds and carries + ``topic``, ``codec``, ``outcome``, and ``error.type`` on failures. +* ``ydb.topic.reader.message.processing.duration`` records time from receipt until each + non-RAW message's batch is ready for delivery. It includes decoder queueing and uses + the same attributes as the compression histogram. +* ``ydb.topic.reader.commit.queued`` counts messages in commit ranges accepted by the + SDK, using ``range.end - range.start``. It carries the Reader attributes and ``topic``. +* ``ydb.topic.reader.commit.acknowledged`` counts monotonic committed-offset advances + reported by regular acknowledgements. Successful transaction commits count their + submitted ranges; duplicate or stale regular acknowledgements and failed transactions + add nothing. It carries the Reader attributes and ``topic``. +* ``ydb.topic.reader.commit_offset.lag.max`` reports the maximum gap between the highest + requested commit end offset and the acknowledged offset across active partitions of + each ``topic``. It tracks regular stream commits; transactional ranges remain visible + in the commit counters but do not contribute to this gauge. Stale acknowledgements + cannot increase it. +* ``ydb.topic.reader.session.errors`` counts failed sessions or connection attempts. + The ``retry_decision`` attribute is ``retry`` when the Reader will reconnect and + ``stop`` when it terminates. ``status_code`` identifies the YDB status or exception + class, while ``error.type`` classifies transport, YDB, and other failures. Normal + Reader shutdown is not counted. This Reader-level metric has no ``topic`` attribute. +* ``ydb.topic.reader.credit_balance_bytes`` reports the protocol byte credit granted to + the server and not yet consumed by read responses. It is ``0`` between stream + sessions, uses unit ``By``, and has the Reader attributes but no ``topic``. * ``ydb.topic.reader.local_buffer.messages`` reports the current number of messages retained by the logical Reader, including messages waiting for decoding and decoded messages waiting for the application. It has the Reader attributes but no ``topic``. @@ -228,9 +253,9 @@ This produces three cumulative counters and three observable instruments: attribute. If ``reader_name`` is omitted or empty, the Reader still works and receives its normal -process-local diagnostic name, but these six metrics are not published. The runnable +process-local diagnostic name, but these thirteen metrics are not published. The runnable ``examples/opentelemetry/otel_example.py`` creates a topic, writes one message, reads it -with an explicitly named Reader, and exports all six metrics through OTLP. +with an explicitly named Reader, and exports its Reader metrics through OTLP. Instrumented Spans and Attributes diff --git a/examples/opentelemetry/Dockerfile b/examples/opentelemetry/Dockerfile index 041eb4abf..961f5f321 100644 --- a/examples/opentelemetry/Dockerfile +++ b/examples/opentelemetry/Dockerfile @@ -20,3 +20,4 @@ RUN pip install --no-cache-dir -e '.[opentelemetry]' -r examples/opentelemetry/r # Demo scripts. COPY examples/opentelemetry/otel_example.py ./examples/opentelemetry/otel_example.py COPY examples/opentelemetry/load_tank.py ./examples/opentelemetry/load_tank.py +COPY examples/opentelemetry/verify_metrics.py ./examples/opentelemetry/verify_metrics.py diff --git a/examples/opentelemetry/README.md b/examples/opentelemetry/README.md index 7653486c0..47593d54e 100644 --- a/examples/opentelemetry/README.md +++ b/examples/opentelemetry/README.md @@ -5,6 +5,13 @@ Async demo in [`otel_example.py`](otel_example.py): OTLP export, `enable_tracing metrics, bank table, Serializable transactions (TLI-style load), and an explicitly named topic Reader that emits `ydb.topic.reader.received.messages`, `ydb.topic.reader.received.bytes`, `ydb.topic.reader.delivered.messages`, +`ydb.topic.reader.compression.duration`, +`ydb.topic.reader.message.processing.duration`, +`ydb.topic.reader.commit.queued`, +`ydb.topic.reader.commit.acknowledged`, +`ydb.topic.reader.commit_offset.lag.max`, +`ydb.topic.reader.session.errors`, +`ydb.topic.reader.credit_balance_bytes`, `ydb.topic.reader.local_buffer.messages`, `ydb.topic.reader.local_buffer.message_age.max`, and `ydb.topic.reader.partition_session.count`. @@ -22,6 +29,26 @@ that topic during cleanup, then writes and reads one message, producing: from several topics; - `ydb.topic.reader.delivered.messages`: the number of messages returned by `receive_message`, `receive_batch`, or `receive_batch_with_tx`, split by `topic`; +- `ydb.topic.reader.compression.duration`: seconds spent decompressing each non-RAW + message, excluding decoder-executor queueing, split by `topic`, `codec`, and + `outcome`; +- `ydb.topic.reader.message.processing.duration`: seconds from receipt until each + non-RAW message's batch is ready for delivery, including decoder queueing and split + by `topic`, `codec`, and `outcome`; +- `ydb.topic.reader.commit.queued`: messages in commit ranges accepted by the SDK, + counted as `range.end - range.start` and split by `topic`; +- `ydb.topic.reader.commit.acknowledged`: monotonic committed-offset advances reported + by regular acknowledgements, plus ranges from successful transaction commits, split + by `topic`; +- `ydb.topic.reader.commit_offset.lag.max`: the largest gap between a regular stream + commit request's end offset and the acknowledged offset across active partitions, + split by `topic`; +- `ydb.topic.reader.session.errors`: failed sessions or connection attempts, split by + `retry_decision=retry|stop`, `status_code`, and `error.type`; normal shutdown is not + counted and `topic` is omitted; +- `ydb.topic.reader.credit_balance_bytes`: protocol byte credit granted to the server + and not yet consumed by read responses; `0` between stream sessions and without + `topic`; - `ydb.topic.reader.local_buffer.messages`: the current number of messages retained by the logical Reader, including decoding and ready-to-deliver queues, without `topic`; - `ydb.topic.reader.local_buffer.message_age.max`: the age in seconds of the oldest @@ -30,7 +57,7 @@ that topic during cleanup, then writes and reads one message, producing: sessions, split by `topic` and without a `partition` label; configured topics report `0` before their first session starts. -All six metrics also carry `endpoint`, `database`, and `reader.name`; `consumer` is +All thirteen metrics also carry `endpoint`, `database`, and `reader.name`; `consumer` is included for consumer-based reads. They remain attached to the same logical Reader across stream reconnects. Use the rate of the cumulative counters to compare incoming traffic with messages visible to the application, and read the observable instruments @@ -48,6 +75,15 @@ metrics dashboard: Peak RPS -> Medium RPS -> Min RPS -> Medium RPS -> repeat ``` +The same process also creates a temporary multi-partition topic and keeps a named +Reader and writer active for the whole run. By default the writer produces 20 messages +per second while the Reader deliberately pauses for 0.1 seconds per message. This +creates a visible, bounded SDK-side prefetch buffer and continuously exercises all +Topic Reader counters, histograms, and gauges. A short Reader against a deliberately missing topic +emits the deterministic `session.errors{retry_decision="stop"}` case without +interrupting the main stream. A synthetic transient `ABORTED` probe exercises real +multi-attempt retry metrics without changing the database. + Most steps assume the **repository root** as the current directory; the install step also shows the variant from this folder. ## 1. Start YDB (or the full stack) with Docker **first** @@ -84,15 +120,44 @@ The first run builds the `otel-example` image from the local SDK source (`Docker Grafana: http://localhost:3000 Prometheus: http://localhost:9090 +To turn this into an automated e2e check, start the stack in the background and run +the verifier. It waits up to three minutes for every SDK metric, including the topic +error probe, to carry the expected non-zero activity in Prometheus. The retry check +also requires at least one operation with more than one attempt. The verifier reads +the current load-generator run ID from the shared Compose volume and only accepts +series carrying that `service.instance.id`, so metrics left by an older run cannot +make the check pass: + +```sh +docker compose -f examples/opentelemetry/compose-e2e.yaml up --build -d +docker compose -f examples/opentelemetry/compose-e2e.yaml --profile verify run --rm metrics-verifier +``` + +The verifier returns exit code `1` and prints the missing Prometheus series if the +contract is incomplete. Run it directly against another Prometheus instance with: + +```sh +python examples/opentelemetry/verify_metrics.py \ + --prometheus-url http://localhost:9090 \ + --service-instance-id \ + --require-error-probe +``` + +Use the value printed as `run_id` by `load_tank.py`, or set `LOAD_TANK_RUN_ID` +explicitly before starting it. The verifier requires either `--service-instance-id` +or `--run-id-file`; it never searches unscoped historical series. + Grafana is provisioned with the **YDB Python SDK Metrics** dashboard. It uses -Prometheus queries for SDK metrics such as `db_client_operation_duration`, +Prometheus queries for SDK metrics such as `ydb_client_operation_duration`, `ydb_client_operation_failed`, `ydb_query_session_count`, `ydb_query_session_pending_requests`, `ydb_query_session_create_time`, and `ydb_client_retry_duration`. The example also publishes the topic Reader received and delivered message counters by topic, the reader-level received-byte counter, and the -reader-level local-buffer count and oldest-age gauges. It also publishes the active -partition-session count by topic. Use Grafana Explore for ad-hoc traces through Tempo -and metrics through Prometheus. +reader-level local-buffer count and oldest-age gauges. It also publishes commit-queued +and commit-acknowledged messages, commit-offset lag, and the active partition-session +count by topic, the reader-level credit balance, and message processing/decompression +latency histograms. Use Grafana Explore for ad-hoc traces through Tempo and metrics +through Prometheus. In Grafana Explore, select the Prometheus data source and query: @@ -103,7 +168,7 @@ ydb_topic_reader_local_buffer_messages{reader_name="otel-example-reader"} The oldest buffered-message age is available as: ```promql -ydb_topic_reader_local_buffer_message_age_max{reader_name="otel-example-reader"} +ydb_topic_reader_local_buffer_message_age_max_seconds{reader_name="otel-example-reader"} ``` Active partition sessions are available as: @@ -112,12 +177,62 @@ Active partition sessions are available as: ydb_topic_reader_partition_session_count{reader_name="otel-example-reader"} ``` -The three counters use Prometheus' conventional `_total` suffix, for example: +The current byte credit granted to the server is available as: + +```promql +ydb_topic_reader_credit_balance_bytes{reader_name="otel-example-reader"} +``` + +The six counters use Prometheus' conventional `_total` suffix, for example: ```promql ydb_topic_reader_received_messages_total{reader_name="otel-example-reader"} ``` +Commit ranges accepted by the Reader are available as: + +```promql +ydb_topic_reader_commit_queued_total{reader_name="otel-example-reader"} +``` + +Successfully acknowledged commit ranges are available as: + +```promql +ydb_topic_reader_commit_acknowledged_total{reader_name="otel-example-reader"} +``` + +The maximum outstanding commit-offset gap is available as: + +```promql +ydb_topic_reader_commit_offset_lag_max{reader_name="otel-example-reader"} +``` + +The p95 time from message receipt until the decoded batch is ready is available as: + +```promql +histogram_quantile(0.95, + sum by (le, reader_name, topic, codec, outcome) ( + rate(ydb_topic_reader_message_processing_duration_seconds_bucket[$__rate_interval]) + ) +) +``` + +The p95 time spent inside the decompressor is available as: + +```promql +histogram_quantile(0.95, + sum by (le, reader_name, topic, codec, outcome) ( + rate(ydb_topic_reader_compression_duration_seconds_bucket[$__rate_interval]) + ) +) +``` + +Session errors appear only after a failed session or connection attempt: + +```promql +ydb_topic_reader_session_errors_total{reader_name="load-tank-topic-error-probe", retry_decision="stop"} +``` + The SDK configures explicit OpenTelemetry histogram bucket boundaries for its own duration and retry-attempt metrics. Duration values are recorded in seconds, with sub-millisecond and millisecond-scale buckets so Grafana percentiles show @@ -174,8 +289,13 @@ python examples/opentelemetry/load_tank.py | `YDB_DATABASE` | default `/local` | | `OTEL_EXPORTER_OTLP_ENDPOINT` | e.g. `http://otel-collector:4317` | | `OTEL_SERVICE_NAME` | service name attached to exported telemetry | +| `LOAD_TANK_RUN_ID` | optional stable `service.instance.id`; a random ID is used for each run | +| `LOAD_TANK_RUN_ID_FILE` | optional file to which the load tank atomically publishes its run ID | +| `LOAD_TANK_READY_FILE` | optional readiness marker created after YDB and the error probe succeed | +| `METRICS_SERVICE_INSTANCE_ID` | run ID selected by `verify_metrics.py` | +| `METRICS_RUN_ID_FILE` | file from which `verify_metrics.py` reads the selected run ID | | `OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS` | seconds to expose each buffer-gauge state, default `7` | -| `LOAD_TANK_TOTAL_TIME` | total load duration in seconds, default `6000` | +| `LOAD_TANK_TOTAL_TIME` | total load duration in seconds, default `300`; Compose uses `6000` | | `LOAD_TANK_WORKERS` | number of concurrent workers, default `40` | | `LOAD_TANK_POOL_SIZE` | query session pool size, default `20` | | `LOAD_TANK_PEAK_RPS` | peak phase target RPS, default `120` | @@ -188,8 +308,57 @@ python examples/opentelemetry/load_tank.py | `LOAD_TANK_PRESSURE_ACQUIRE_TIMEOUT` | short acquire timeout for timeout metrics, default `1.0` | | `LOAD_TANK_PRESSURE_INTERVAL` | pause between pressure rounds, default `0.2` | | `LOAD_TANK_SESSION_CHURN_INTERVAL` | interval for creating fresh sessions, default `2.0` | +| `LOAD_TANK_RETRY_PROBE_INTERVAL` | interval between deterministic retry probes, default `5.0`; set `0` to disable | +| `LOAD_TANK_RETRY_PROBE_FAILURES` | transient `ABORTED` attempts before success, default `2` | +| `LOAD_TANK_TOPIC_WRITE_RPS` | topic producer rate, default `20`; set `0` to disable topic load | +| `LOAD_TANK_TOPIC_READ_DELAY` | delay after each topic message, default `0.1` seconds | +| `LOAD_TANK_TOPIC_MESSAGE_SIZE` | topic payload size, default `1024` bytes | +| `LOAD_TANK_TOPIC_PARTITIONS` | temporary topic partition count, default `2` | +| `LOAD_TANK_TOPIC_READER_NAME` | stable Reader metric label, default `load-tank-topic-reader` | +| `LOAD_TANK_TOPIC_ERROR_PROBE_ENDPOINT` | endpoint used for a missing-topic error probe; defaults to `YDB_ENDPOINT` | +| `LOAD_TANK_TOPIC_ERROR_PROBE_SECONDS` | probe duration, default `0`; Compose sets it to `5` | | `LOAD_TANK_PEAK_DURATION` | peak phase duration in seconds, default `60` | | `LOAD_TANK_MEDIUM_DURATION` | medium phase duration in seconds, default `90` | | `LOAD_TANK_MIN_DURATION` | low phase duration in seconds, default `60` | | `LOAD_TANK_QUERY` | query executed by workers, default `SELECT 1 AS value` | | `LOAD_TANK_ERROR_QUERY` | query used to produce failed-operation metrics | + +## Load-test recipes + +Run only the infrastructure first, then start one or more isolated load-generator +containers with overrides. This makes test duration and intensity explicit and keeps +Grafana available after a run: + +```sh +docker compose -f examples/opentelemetry/compose-e2e.yaml up -d \ + ydb otel-collector prometheus tempo grafana + +# Five-minute smoke/load run. +docker compose -f examples/opentelemetry/compose-e2e.yaml run --rm \ + -e LOAD_TANK_TOTAL_TIME=300 \ + -e LOAD_TANK_PEAK_RPS=200 \ + -e LOAD_TANK_WORKERS=80 \ + load-generator +``` + +Useful scenarios: + +- **Throughput staircase:** increase `PEAK_RPS` until achieved RPS stops following the + target or p95/p99 latency bends sharply. +- **Pool saturation:** keep `LOAD_TANK_PRESSURE_POOL_SIZE=1`, increase + `LOAD_TANK_PRESSURE_WORKERS`, and watch pending requests and acquire timeouts. +- **Retries:** increase `LOAD_TANK_RETRY_PROBE_FAILURES` and watch retry-attempt count, + average attempts, retrying calls, and retry duration. +- **Topic backpressure:** make `1 / LOAD_TANK_TOPIC_READ_DELAY` lower than + `LOAD_TANK_TOPIC_WRITE_RPS`; the local-buffer count and oldest-message age should + rise while received and delivered rates diverge. +- **Balanced topic stream:** set read capacity above write RPS; the buffer and age + should return close to zero while received and delivered rates converge. +- **Reconnect/chaos:** use the repository's `tests/slo` multi-node chaos setup for a + long run. The Reader session-error counter should grow while message delivery + resumes; do not restart this example's single in-memory YDB merely to create an + error sample. + +For repeatable comparison, change one dimension at a time, warm up before measuring, +keep a fixed measurement window, and record offered RPS, achieved RPS, p50/p95/p99, +error rate, retry ratio, pool saturation, and topic receive/delivery lag. diff --git a/examples/opentelemetry/compose-e2e.yaml b/examples/opentelemetry/compose-e2e.yaml index 325d00c2a..59f46a6f9 100644 --- a/examples/opentelemetry/compose-e2e.yaml +++ b/examples/opentelemetry/compose-e2e.yaml @@ -12,6 +12,7 @@ services: image: ydbplatform/local-ydb:trunk restart: always platform: linux/amd64 + hostname: localhost environment: YDB_DEFAULT_LOG_LEVEL: NOTICE GRPC_TLS_PORT: "2135" @@ -96,12 +97,19 @@ services: build: context: ../.. dockerfile: examples/opentelemetry/Dockerfile - command: ["python", "examples/opentelemetry/load_tank.py"] + command: + [ + "sh", + "-c", + "rm -f /tmp/ydb-load-tank-ready && exec python examples/opentelemetry/load_tank.py", + ] environment: YDB_ENDPOINT: grpc://ydb:2136 YDB_DATABASE: /local OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 OTEL_SERVICE_NAME: ydb-python-load-tank + LOAD_TANK_RUN_ID_FILE: /run/ydb-metrics/run-id + LOAD_TANK_READY_FILE: /tmp/ydb-load-tank-ready LOAD_TANK_TOTAL_TIME: "6000" LOAD_TANK_WORKERS: "40" LOAD_TANK_POOL_SIZE: "20" @@ -115,9 +123,51 @@ services: LOAD_TANK_PRESSURE_ACQUIRE_TIMEOUT: "1.0" LOAD_TANK_PRESSURE_INTERVAL: "0.2" LOAD_TANK_SESSION_CHURN_INTERVAL: "2.0" + LOAD_TANK_RETRY_PROBE_INTERVAL: "5.0" + LOAD_TANK_RETRY_PROBE_FAILURES: "2" + LOAD_TANK_TOPIC_WRITE_RPS: "20" + LOAD_TANK_TOPIC_READ_DELAY: "0.1" + LOAD_TANK_TOPIC_MESSAGE_SIZE: "1024" + LOAD_TANK_TOPIC_PARTITIONS: "2" + LOAD_TANK_TOPIC_ERROR_PROBE_ENDPOINT: grpc://ydb:2136 + LOAD_TANK_TOPIC_ERROR_PROBE_SECONDS: "5" + volumes: + - metrics-run:/run/ydb-metrics depends_on: ydb: condition: service_healthy otel-collector: condition: service_started + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/ydb-load-tank-ready"] + interval: 1s + timeout: 1s + retries: 90 restart: "no" + + metrics-verifier: + profiles: ["verify"] + build: + context: ../.. + dockerfile: examples/opentelemetry/Dockerfile + command: + [ + "python", + "examples/opentelemetry/verify_metrics.py", + "--prometheus-url", + "http://prometheus:9090", + "--require-error-probe", + ] + environment: + METRICS_RUN_ID_FILE: /run/ydb-metrics/run-id + volumes: + - metrics-run:/run/ydb-metrics:ro + depends_on: + prometheus: + condition: service_started + load-generator: + condition: service_healthy + restart: "no" + +volumes: + metrics-run: diff --git a/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json b/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json index 1a0706aa8..6d612101c 100644 --- a/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json +++ b/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json @@ -14,7 +14,7 @@ "options": { "legend": { "displayMode": "table", "placement": "right" } }, "targets": [ { - "expr": "sum by (ydb_operation_name) (rate(db_client_operation_duration_seconds_count[$__rate_interval]))", + "expr": "sum by (ydb_operation_name) (rate(ydb_client_operation_duration_seconds_count[$__rate_interval]))", "legendFormat": "{{ydb_operation_name}}", "datasource": { "type": "prometheus", "uid": "prometheus" } } @@ -99,17 +99,17 @@ "options": { "legend": { "displayMode": "table", "placement": "right" } }, "targets": [ { - "expr": "histogram_quantile(0.50, sum by (le, ydb_operation_name) (rate(db_client_operation_duration_seconds_bucket[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, ydb_operation_name) (rate(ydb_client_operation_duration_seconds_bucket[$__rate_interval])))", "legendFormat": "p50 - {{ydb_operation_name}}", "datasource": { "type": "prometheus", "uid": "prometheus" } }, { - "expr": "histogram_quantile(0.95, sum by (le, ydb_operation_name) (rate(db_client_operation_duration_seconds_bucket[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le, ydb_operation_name) (rate(ydb_client_operation_duration_seconds_bucket[$__rate_interval])))", "legendFormat": "p95 - {{ydb_operation_name}}", "datasource": { "type": "prometheus", "uid": "prometheus" } }, { - "expr": "histogram_quantile(0.99, sum by (le, ydb_operation_name) (rate(db_client_operation_duration_seconds_bucket[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, ydb_operation_name) (rate(ydb_client_operation_duration_seconds_bucket[$__rate_interval])))", "legendFormat": "p99 - {{ydb_operation_name}}", "datasource": { "type": "prometheus", "uid": "prometheus" } } @@ -186,6 +186,175 @@ "datasource": { "type": "prometheus", "uid": "prometheus" } } ] + }, + { + "title": "Closed Query Sessions", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "sum by (ydb_query_session_pool_name, reason) (increase(ydb_query_session_closed_total[$__rate_interval]))", + "legendFormat": "{{ydb_query_session_pool_name}} - {{reason}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Reader Session Errors", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, retry_decision) (increase(ydb_topic_reader_session_errors_total[$__rate_interval]))", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{retry_decision}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Message Rate", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, + "fieldConfig": { "defaults": { "unit": "msgps" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_received_messages_total[$__rate_interval]))", + "legendFormat": "received - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + }, + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_delivered_messages_total[$__rate_interval]))", + "legendFormat": "delivered - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + }, + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_commit_queued_total[$__rate_interval]))", + "legendFormat": "commit queued - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + }, + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_commit_acknowledged_total[$__rate_interval]))", + "legendFormat": "commit acknowledged - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Received Throughput", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, + "fieldConfig": { "defaults": { "unit": "Bps" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name) (rate(ydb_topic_reader_received_bytes_total[$__rate_interval]))", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Reader Local Buffer", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 48 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name) (ydb_topic_reader_local_buffer_messages)", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Oldest Buffered Topic Message", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 48 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "max by (service_instance_id, endpoint, database, consumer, reader_name) (ydb_topic_reader_local_buffer_message_age_max_seconds)", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Partition Sessions", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 48 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (ydb_topic_reader_partition_session_count)", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Commit Offset Lag", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "max by (service_instance_id, endpoint, database, consumer, reader_name, topic) (ydb_topic_reader_commit_offset_lag_max)", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Reader Credit Balance", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name) (ydb_topic_reader_credit_balance_bytes)", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Message Processing p95", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id, endpoint, database, consumer, reader_name, topic, codec, outcome) (rate(ydb_topic_reader_message_processing_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}} - {{codec}} - {{outcome}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] + }, + { + "title": "Topic Decompression p95", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "options": { "legend": { "displayMode": "table", "placement": "right" } }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id, endpoint, database, consumer, reader_name, topic, codec, outcome) (rate(ydb_topic_reader_compression_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}} - {{codec}} - {{outcome}}", + "datasource": { "type": "prometheus", "uid": "prometheus" } + } + ] } ] } diff --git a/examples/opentelemetry/load_tank.py b/examples/opentelemetry/load_tank.py index 801ddadf5..7fccbf134 100644 --- a/examples/opentelemetry/load_tank.py +++ b/examples/opentelemetry/load_tank.py @@ -6,13 +6,10 @@ import os import random import time +import uuid from dataclasses import dataclass -from typing import AsyncIterator, Tuple - -from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics._internal.export import PeriodicExportingMetricReader -from opentelemetry.sdk.resources import Resource +from pathlib import Path +from typing import Any, AsyncIterator, Awaitable, Dict, Optional, Tuple import ydb from ydb.opentelemetry import enable_metrics @@ -24,6 +21,9 @@ class LoadConfig: database: str otlp_endpoint: str service_name: str + run_id: str + run_id_file: str + ready_file: str pool_size: int worker_count: int peak_rps: int @@ -42,6 +42,15 @@ class LoadConfig: pressure_acquire_timeout: float pressure_interval: float session_churn_interval: float + retry_probe_interval: float + retry_probe_failures: int + topic_write_rps: int + topic_read_delay: float + topic_message_size: int + topic_partitions: int + topic_reader_name: str + topic_error_probe_endpoint: str + topic_error_probe_seconds: float def _env(name: str, default: str) -> str: @@ -63,6 +72,9 @@ def _load_config() -> LoadConfig: database=_env("YDB_DATABASE", "/local"), otlp_endpoint=_env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317"), service_name=_env("OTEL_SERVICE_NAME", "ydb-python-load-tank"), + run_id=_env("LOAD_TANK_RUN_ID", uuid.uuid4().hex), + run_id_file=_env("LOAD_TANK_RUN_ID_FILE", ""), + ready_file=_env("LOAD_TANK_READY_FILE", ""), pool_size=_env_int("LOAD_TANK_POOL_SIZE", 20), worker_count=_env_int("LOAD_TANK_WORKERS", 40), peak_rps=_env_int("LOAD_TANK_PEAK_RPS", 120), @@ -81,9 +93,94 @@ def _load_config() -> LoadConfig: pressure_acquire_timeout=_env_float("LOAD_TANK_PRESSURE_ACQUIRE_TIMEOUT", 1.0), pressure_interval=_env_float("LOAD_TANK_PRESSURE_INTERVAL", 0.2), session_churn_interval=_env_float("LOAD_TANK_SESSION_CHURN_INTERVAL", 2.0), + retry_probe_interval=_env_float("LOAD_TANK_RETRY_PROBE_INTERVAL", 5.0), + retry_probe_failures=_env_int("LOAD_TANK_RETRY_PROBE_FAILURES", 2), + topic_write_rps=_env_int("LOAD_TANK_TOPIC_WRITE_RPS", 20), + topic_read_delay=_env_float("LOAD_TANK_TOPIC_READ_DELAY", 0.1), + topic_message_size=_env_int("LOAD_TANK_TOPIC_MESSAGE_SIZE", 1024), + topic_partitions=_env_int("LOAD_TANK_TOPIC_PARTITIONS", 2), + topic_reader_name=_env("LOAD_TANK_TOPIC_READER_NAME", "load-tank-topic-reader"), + topic_error_probe_endpoint=_env( + "LOAD_TANK_TOPIC_ERROR_PROBE_ENDPOINT", + _env("YDB_ENDPOINT", "grpc://localhost:2136"), + ), + topic_error_probe_seconds=_env_float("LOAD_TANK_TOPIC_ERROR_PROBE_SECONDS", 0), ) +def _publish_run_id(run_id: str, run_id_file: str, ready_file: str) -> None: + if ready_file: + Path(ready_file).unlink(missing_ok=True) + + if run_id_file: + target = Path(run_id_file) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name("%s.%s.tmp" % (target.name, uuid.uuid4().hex)) + temporary.write_text("%s\n" % run_id, encoding="utf-8") + os.replace(temporary, target) + + if ready_file: + target = Path(ready_file) + target.parent.mkdir(parents=True, exist_ok=True) + target.touch() + + +async def _await_with_background( + awaitable: Awaitable[Any], + background_tasks: Dict[asyncio.Task, str], +) -> Any: + operation_task = asyncio.ensure_future(awaitable) + try: + while True: + done, _ = await asyncio.wait( + {operation_task, *background_tasks}, + return_when=asyncio.FIRST_COMPLETED, + ) + for task in done: + if task is operation_task: + continue + name = background_tasks[task] + await _raise_background_failure(task, name) + + if operation_task in done: + return await operation_task + finally: + if not operation_task.done(): + operation_task.cancel() + await asyncio.gather(operation_task, return_exceptions=True) + + +async def _raise_background_failure(task: asyncio.Task, name: str) -> None: + try: + await task + except asyncio.CancelledError as error: + raise RuntimeError("%s was cancelled unexpectedly" % name) from error + except Exception as error: + raise RuntimeError("%s failed" % name) from error + raise RuntimeError("%s stopped unexpectedly" % name) + + +async def _cancel_background_tasks( + background_tasks: Dict[asyncio.Task, str], + excluded_task: Optional[asyncio.Task] = None, + raise_failures: bool = False, +) -> None: + tasks = [(task, name) for task, name in background_tasks.items() if task is not excluded_task] + unexpectedly_done = [(task, name) for task, name in tasks if task.done()] + + for task, _ in tasks: + task.cancel() + results = await asyncio.gather(*(task for task, _ in tasks), return_exceptions=True) + + if not raise_failures: + return + if unexpectedly_done: + await _raise_background_failure(*unexpectedly_done[0]) + for (task, name), result in zip(tasks, results): + if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError): + await _raise_background_failure(task, name) + + async def _load_steps(config: LoadConfig) -> AsyncIterator[Tuple[int, str, int]]: pattern = ( (config.peak_rps, "Peak", config.peak_duration), @@ -146,7 +243,7 @@ async def _error_worker(pool: ydb.aio.QuerySessionPool, config: LoadConfig, stop while not stop.is_set(): try: await pool.execute_with_retries(config.error_query) - except Exception: + except ydb.SchemeError: pass next_tick += interval @@ -169,13 +266,13 @@ async def contender() -> None: try: async with pool.checkout(timeout=config.pressure_acquire_timeout): pass - except Exception: + except ydb.SessionPoolEmpty: pass holder_task = asyncio.create_task(holder()) await asyncio.sleep(0) contenders = [asyncio.create_task(contender()) for _ in range(config.pressure_workers)] - await asyncio.gather(holder_task, *contenders, return_exceptions=True) + await asyncio.gather(holder_task, *contenders) async def _pool_pressure_worker(driver: ydb.aio.Driver, config: LoadConfig, stop: asyncio.Event) -> None: @@ -209,10 +306,147 @@ async def _session_churn_worker(driver: ydb.aio.Driver, config: LoadConfig, stop pass +async def _retry_probe_worker(config: LoadConfig, stop: asyncio.Event) -> None: + if config.retry_probe_interval <= 0 or config.retry_probe_failures <= 0: + return + + retry_settings = ydb.RetrySettings(max_retries=config.retry_probe_failures) + while not stop.is_set(): + attempt = 0 + + async def transient_failure() -> None: + nonlocal attempt + attempt += 1 + if attempt <= config.retry_probe_failures: + raise ydb.Aborted("load-tank retry probe") + + await ydb.retry_operation_async(transient_failure, retry_settings) + try: + await asyncio.wait_for(stop.wait(), timeout=config.retry_probe_interval) + except asyncio.TimeoutError: + pass + + +def _topic_payload(sequence_number: int, size: int) -> bytes: + prefix = "%s:%s" % (sequence_number, time.time_ns()) + return prefix.encode("ascii").ljust(max(size, 1), b"x")[: max(size, 1)] + + +async def _topic_writer(writer, config: LoadConfig, stop: asyncio.Event) -> None: + interval = 1.0 / config.topic_write_rps + next_tick = time.monotonic() + sequence_number = 0 + + while not stop.is_set(): + await writer.write(_topic_payload(sequence_number, config.topic_message_size)) + sequence_number += 1 + next_tick += interval + delay = next_tick - time.monotonic() + if delay > 0: + try: + await asyncio.wait_for(stop.wait(), timeout=delay) + except asyncio.TimeoutError: + pass + else: + await asyncio.sleep(0) + + +async def _topic_reader(reader, config: LoadConfig, stop: asyncio.Event) -> None: + while not stop.is_set(): + try: + message = await asyncio.wait_for(reader.receive_message(), timeout=1) + except asyncio.TimeoutError: + continue + + if config.topic_read_delay > 0: + try: + await asyncio.wait_for(stop.wait(), timeout=config.topic_read_delay) + except asyncio.TimeoutError: + pass + await reader.commit_with_ack(message) + + +async def _topic_stream_worker(driver: ydb.aio.Driver, config: LoadConfig, stop: asyncio.Event) -> None: + if config.topic_write_rps <= 0: + return + + topic_path = "%s/otel-load-%s" % (config.database.rstrip("/"), uuid.uuid4().hex) + consumer = "load-tank-consumer" + await driver.topic_client.create_topic( + topic_path, + min_active_partitions=config.topic_partitions, + partition_count_limit=config.topic_partitions, + consumers=[consumer], + ) + print( + "Topic stream: path=%s writer=%s RPS reader_delay=%ss message_size=%sB partitions=%s" + % ( + topic_path, + config.topic_write_rps, + config.topic_read_delay, + config.topic_message_size, + config.topic_partitions, + ) + ) + + try: + async with driver.topic_client.writer(topic_path, codec=ydb.TopicCodec.GZIP) as writer: + async with driver.topic_client.reader( + topic_path, + consumer, + reader_name=config.topic_reader_name, + ) as reader: + writer_task = asyncio.create_task(_topic_writer(writer, config, stop)) + reader_task = asyncio.create_task(_topic_reader(reader, config, stop)) + try: + await asyncio.gather(writer_task, reader_task) + finally: + writer_task.cancel() + reader_task.cancel() + await asyncio.gather(writer_task, reader_task, return_exceptions=True) + finally: + await driver.topic_client.drop_topic(topic_path) + + +async def _topic_error_probe(config: LoadConfig) -> None: + if config.topic_error_probe_seconds <= 0: + return + + topic_path = "%s/otel-unreachable" % config.database.rstrip("/") + try: + async with ydb.aio.Driver( + endpoint=config.topic_error_probe_endpoint, + database=config.database, + disable_discovery=True, + ) as driver: + async with driver.topic_client.reader( + topic_path, + "load-tank-error-probe", + reader_name="load-tank-topic-error-probe", + ) as reader: + await asyncio.wait_for(reader.wait_message(), timeout=config.topic_error_probe_seconds) + except ydb.SchemeError as error: + print("Topic error probe produced the expected session failure: %s" % type(error).__name__) + except asyncio.TimeoutError as error: + raise RuntimeError("Topic error probe did not produce SCHEME_ERROR before its timeout") from error + else: + raise RuntimeError("Topic error probe unexpectedly received a message") + + async def main() -> None: + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics._internal.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + config = _load_config() - resource = Resource(attributes={"service.name": config.service_name}) + resource = Resource( + attributes={ + "service.name": config.service_name, + "service.instance.id": config.run_id, + } + ) metric_reader = PeriodicExportingMetricReader( OTLPMetricExporter(endpoint=config.otlp_endpoint), export_interval_millis=2000, @@ -222,11 +456,15 @@ async def main() -> None: print( "=== YDB Python SDK load tank ===\n" - " total=%ss workers=%s pool_size=%s query=%r error_rps=%s\n" + " run_id=%s total=%ss workers=%s pool_size=%s query=%r error_rps=%s\n" " pressure_pool_size=%s pressure_workers=%s pressure_timeout=%ss session_churn_interval=%ss\n" + " retry_probe_interval=%ss retry_probe_failures=%s\n" + " topic_write_rps=%s topic_read_delay=%ss topic_message_size=%sB topic_partitions=%s\n" + " topic_error_probe=%ss at %s\n" " pattern: Peak(%s rps, %ss) -> Medium(%s rps, %ss) -> " "Min(%s rps, %ss) -> Medium -> repeat" % ( + config.run_id, config.total_time, config.worker_count, config.pool_size, @@ -236,6 +474,14 @@ async def main() -> None: config.pressure_workers, config.pressure_acquire_timeout, config.session_churn_interval, + config.retry_probe_interval, + config.retry_probe_failures, + config.topic_write_rps, + config.topic_read_delay, + config.topic_message_size, + config.topic_partitions, + config.topic_error_probe_seconds, + config.topic_error_probe_endpoint, config.peak_rps, config.peak_duration, config.medium_rps, @@ -245,42 +491,85 @@ async def main() -> None: ) ) - async with ydb.aio.Driver( - endpoint=config.endpoint, - database=config.database, - disable_discovery=True, - ) as driver: - await driver.wait(timeout=60) - - async with ydb.aio.QuerySessionPool(driver, size=config.pool_size, name="load-tank") as pool: - queue: asyncio.Queue[object] = asyncio.Queue(maxsize=max(config.worker_count * 4, config.peak_rps)) - stop = asyncio.Event() - workers = [ - asyncio.create_task(_worker(pool, queue, config.query, stop)) for _ in range(config.worker_count) - ] - error_task = asyncio.create_task(_error_worker(pool, config, stop)) - pressure_task = asyncio.create_task(_pool_pressure_worker(driver, config, stop)) - churn_task = asyncio.create_task(_session_churn_worker(driver, config, stop)) - - try: - async for rps, label, duration in _load_steps(config): - print("[%s] Phase: %s (%s RPS for %ss)" % (time.strftime("%H:%M:%S"), label, rps, duration)) - await _feed_phase(queue, rps, duration) - await asyncio.sleep(random.random() / 10.0) - - await queue.join() - finally: - stop.set() - for worker in workers: - worker.cancel() - error_task.cancel() - pressure_task.cancel() - churn_task.cancel() - await asyncio.gather(*workers, error_task, pressure_task, churn_task, return_exceptions=True) - - print("Waiting 10s to flush metrics...") - await asyncio.sleep(10) - meter_provider.shutdown() + try: + async with ydb.aio.Driver( + endpoint=config.endpoint, + database=config.database, + disable_discovery=True, + ) as driver: + await driver.wait(timeout=60) + await _topic_error_probe(config) + _publish_run_id(config.run_id, config.run_id_file, config.ready_file) + + async with ydb.aio.QuerySessionPool(driver, size=config.pool_size, name="load-tank") as pool: + queue: asyncio.Queue[object] = asyncio.Queue(maxsize=max(config.worker_count * 4, config.peak_rps)) + stop = asyncio.Event() + background_tasks: Dict[asyncio.Task, str] = {} + + def start(coroutine: Awaitable[None], name: str) -> asyncio.Task: + task = asyncio.create_task(coroutine) + background_tasks[task] = name + return task + + for index in range(config.worker_count): + start(_worker(pool, queue, config.query, stop), "query worker %s" % index) + if config.error_rps > 0: + start(_error_worker(pool, config, stop), "query error worker") + if config.pressure_workers > 0 and config.pressure_pool_size > 0: + start(_pool_pressure_worker(driver, config, stop), "pool pressure worker") + if config.session_churn_interval > 0: + start(_session_churn_worker(driver, config, stop), "session churn worker") + if config.retry_probe_interval > 0 and config.retry_probe_failures > 0: + start(_retry_probe_worker(config, stop), "retry probe worker") + topic_task = None + if config.topic_write_rps > 0: + topic_task = start(_topic_stream_worker(driver, config, stop), "topic stream worker") + + completed = False + try: + async for rps, label, duration in _load_steps(config): + print("[%s] Phase: %s (%s RPS for %ss)" % (time.strftime("%H:%M:%S"), label, rps, duration)) + await _await_with_background(_feed_phase(queue, rps, duration), background_tasks) + await _await_with_background(asyncio.sleep(random.random() / 10.0), background_tasks) + + await _await_with_background(queue.join(), background_tasks) + completed = True + finally: + topic_was_done = topic_task is not None and topic_task.done() + stop.set() + background_failure = None + try: + await _cancel_background_tasks( + background_tasks, + excluded_task=topic_task, + raise_failures=completed, + ) + except Exception as error: + if not completed: + raise + background_failure = error + + if topic_task is not None: + try: + await asyncio.wait_for(topic_task, timeout=30) + except asyncio.TimeoutError: + topic_task.cancel() + await asyncio.gather(topic_task, return_exceptions=True) + if completed: + raise RuntimeError("Topic load did not stop within 30 seconds") + except Exception: + if completed: + raise + else: + if completed and topic_was_done: + raise RuntimeError("Topic stream worker stopped unexpectedly") + if background_failure is not None: + raise background_failure + + print("Waiting 10s to flush metrics...") + await asyncio.sleep(10) + finally: + meter_provider.shutdown() if __name__ == "__main__": diff --git a/examples/opentelemetry/otel_example.py b/examples/opentelemetry/otel_example.py index 190046991..92e3d9702 100644 --- a/examples/opentelemetry/otel_example.py +++ b/examples/opentelemetry/otel_example.py @@ -55,7 +55,7 @@ async def _topic_reader_metrics_demo( await driver.topic_client.create_topic(topic_path, consumers=[consumer]) try: - async with driver.topic_client.writer(topic_path) as writer: + async with driver.topic_client.writer(topic_path, codec=ydb.TopicCodec.GZIP) as writer: await writer.write("OpenTelemetry topic Reader metrics") async with driver.topic_client.reader( diff --git a/tests/observability/test_metrics.py b/tests/observability/test_metrics.py index aa4059232..aaa7be862 100644 --- a/tests/observability/test_metrics.py +++ b/tests/observability/test_metrics.py @@ -92,6 +92,15 @@ def _topic_reader_batch(message_count, topic, partition_session_id=1, reader_str ) +def _topic_reader_partition_session(topic, partition_session_id, reader_stream_id, committed_offset=0): + return SimpleNamespace( + topic_path=topic, + id=partition_session_id, + reader_stream_id=reader_stream_id, + committed_offset=committed_offset, + ) + + def _topic_read_response(partition_messages, bytes_size=30): import datetime @@ -144,7 +153,7 @@ def _histogram_boundaries_advisory_supported(): def test_metrics_build_info_version(metrics_setup): from ydb.observability.metrics import _metrics_build_info_tokens - assert _metrics_build_info_tokens() == ["ydb-sdk-metrics/0.7.0"] + assert _metrics_build_info_tokens() == ["ydb-sdk-metrics/0.14.0"] def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): @@ -161,12 +170,19 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): QUERY_SESSION_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DURATION, + TOPIC_READER_COMMIT_ACKNOWLEDGED, + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_COMMIT_QUEUED, + TOPIC_READER_COMPRESSION_DURATION, + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_DELIVERED_MESSAGES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_MESSAGE_PROCESSING_DURATION, TOPIC_READER_PARTITION_SESSION_COUNT, TOPIC_READER_RECEIVED_BYTES, TOPIC_READER_RECEIVED_MESSAGES, + TOPIC_READER_SESSION_ERRORS, ATTEMPT_BUCKETS, DURATION_BUCKETS_SECONDS, RETRY_DURATION_BUCKETS_SECONDS, @@ -209,7 +225,15 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): reader_metrics.record_received_messages([_topic_reader_batch(2, "/Root/events")]) reader_metrics.record_received_bytes(128) reader_metrics.record_delivered_messages(2, "/Root/events") - reader_metrics.add_partition_session("/Root/events") + reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events") + reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events") + reader_metrics.record_commit_queued(2, "/Root/events") + reader_metrics.record_commit_acknowledged(2, "/Root/events") + reader_metrics.record_session_error(issues.Unavailable("retry"), True) + reader_metrics.reset_credit_balance_bytes(1024) + partition_session = _topic_reader_partition_session("/Root/events", 1, 1) + reader_metrics.add_partition_session("/Root/events", partition_session) + reader_metrics.record_commit_offset_queued(partition_session, 2) metrics = _metrics_by_name(metrics_setup) @@ -225,12 +249,19 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): QUERY_SESSION_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DURATION, + TOPIC_READER_COMMIT_ACKNOWLEDGED, + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_COMMIT_QUEUED, + TOPIC_READER_COMPRESSION_DURATION, + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_DELIVERED_MESSAGES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_MESSAGE_PROCESSING_DURATION, TOPIC_READER_PARTITION_SESSION_COUNT, TOPIC_READER_RECEIVED_BYTES, TOPIC_READER_RECEIVED_MESSAGES, + TOPIC_READER_SESSION_ERRORS, } assert metrics[CLIENT_OPERATION_DURATION].unit == "s" assert metrics[CLIENT_OPERATION_FAILED].unit == "{command}" @@ -243,12 +274,21 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): assert metrics[QUERY_SESSION_TIMEOUTS].unit == "{connection}" assert metrics[RETRY_DURATION].unit == "s" assert metrics[RETRY_ATTEMPTS].unit == "{attempt}" + assert metrics[TOPIC_READER_COMMIT_ACKNOWLEDGED].unit == "{message}" + assert metrics[TOPIC_READER_COMMIT_OFFSET_LAG_MAX].unit == "{message}" + assert metrics[TOPIC_READER_COMMIT_QUEUED].unit == "{message}" + assert metrics[TOPIC_READER_COMPRESSION_DURATION].unit == "s" + assert metrics[TOPIC_READER_CREDIT_BALANCE_BYTES].unit == "By" assert metrics[TOPIC_READER_DELIVERED_MESSAGES].unit == "{message}" assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX].unit == "s" assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGES].unit == "{message}" + assert metrics[TOPIC_READER_MESSAGE_PROCESSING_DURATION].unit == "s" assert metrics[TOPIC_READER_PARTITION_SESSION_COUNT].unit == "{session}" assert metrics[TOPIC_READER_RECEIVED_MESSAGES].unit == "{message}" assert metrics[TOPIC_READER_RECEIVED_BYTES].unit == "By" + assert metrics[TOPIC_READER_SESSION_ERRORS].unit == "{error}" + assert metrics[TOPIC_READER_CREDIT_BALANCE_BYTES].data.__class__.__name__ == "Gauge" + assert metrics[TOPIC_READER_COMMIT_OFFSET_LAG_MAX].data.__class__.__name__ == "Gauge" assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX].data.__class__.__name__ == "Gauge" assert metrics[TOPIC_READER_PARTITION_SESSION_COUNT].data.__class__.__name__ == "Sum" if _histogram_boundaries_advisory_supported(): @@ -260,6 +300,14 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch): ) assert _single_point_from_metrics(metrics, RETRY_DURATION).explicit_bounds == RETRY_DURATION_BUCKETS_SECONDS assert _single_point_from_metrics(metrics, RETRY_ATTEMPTS).explicit_bounds == ATTEMPT_BUCKETS + assert ( + _single_point_from_metrics(metrics, TOPIC_READER_COMPRESSION_DURATION).explicit_bounds + == DURATION_BUCKETS_SECONDS + ) + assert ( + _single_point_from_metrics(metrics, TOPIC_READER_MESSAGE_PROCESSING_DURATION).explicit_bounds + == DURATION_BUCKETS_SECONDS + ) def test_metrics_registry_supports_old_histogram_api(): @@ -400,6 +448,15 @@ def fail_on_iteration(): assert noop_reader_metrics.capture_received_at() is None noop_reader_metrics.record_received_bytes(10) noop_reader_metrics.record_delivered_messages(1, "/Root/events") + noop_reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events") + noop_reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events") + noop_reader_metrics.record_commit_queued(1, "/Root/events") + noop_reader_metrics.record_commit_acknowledged(1, "/Root/events") + noop_reader_metrics.record_commit_offset_queued(object(), 1) + noop_reader_metrics.record_commit_offset_acknowledged(object(), 1) + noop_reader_metrics.record_session_error(RuntimeError("ignored"), True) + noop_reader_metrics.add_credit_balance_bytes(-10) + noop_reader_metrics.reset_credit_balance_bytes(10) noop_reader_metrics.remove_local_buffer_messages(1, object()) noop_reader_metrics.reset_local_buffer_messages() noop_reader_metrics.add_partition_session("/Root/events") @@ -1004,6 +1061,8 @@ def test_reconfigure_metrics_quiesces_old_gauge_callbacks(self): def test_topic_reader_gauges_keep_state_while_metrics_are_reconfigured(self, monkeypatch): from ydb.observability.metrics import ( + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, TOPIC_READER_PARTITION_SESSION_COUNT, @@ -1024,12 +1083,17 @@ def test_topic_reader_gauges_keep_state_while_metrics_are_reconfigured(self, mon [first_batch], received_at=10.0, ) - reader_metrics.add_partition_session("/Root/events") + reader_metrics.reset_credit_balance_bytes(100) + partition_session = _topic_reader_partition_session("/Root/events", 1, 1) + reader_metrics.add_partition_session("/Root/events", partition_session) + reader_metrics.record_commit_offset_queued(partition_session, 5) monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=30.0)) try: assert _single_point(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 2 assert _single_point(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0 assert _single_point(first_reader, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1 + assert _single_point(first_reader, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 100 + assert _single_point(first_reader, TOPIC_READER_COMMIT_OFFSET_LAG_MAX).value == 5 disable_metrics() second_batch = _topic_reader_batch(3, "/Root/events") @@ -1038,13 +1102,18 @@ def test_topic_reader_gauges_keep_state_while_metrics_are_reconfigured(self, mon received_at=20.0, ) reader_metrics.remove_local_buffer_messages(1, first_batch._partition_session) + reader_metrics.add_credit_balance_bytes(-20) + reader_metrics.record_commit_offset_acknowledged(partition_session, 2) reader_metrics.add_partition_session("/Root/events") enable_metrics(second_provider) assert _points(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == [] + assert _points(first_reader, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) == [] assert _single_point(second_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 4 assert _single_point(second_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0 assert _single_point(second_reader, TOPIC_READER_PARTITION_SESSION_COUNT).value == 2 + assert _single_point(second_reader, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 80 + assert _single_point(second_reader, TOPIC_READER_COMMIT_OFFSET_LAG_MAX).value == 3 finally: reader_metrics.close() disable_metrics() @@ -1372,6 +1441,353 @@ class FakeDriver: } +def test_topic_reader_processing_histograms_record_success_and_error_per_message(metrics_setup): + from tests.observability.conftest import FakeDriverConfig + from ydb.observability.metrics import ( + TOPIC_READER_COMPRESSION_DURATION, + TOPIC_READER_MESSAGE_PROCESSING_DURATION, + create_topic_reader_metrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + reader_metrics = create_topic_reader_metrics( + FakeDriver(), + consumer_name="analytics", + reader_name="payments-worker", + ) + error = ValueError("cannot decode") + reader_metrics.record_compression_duration(0.01, "gzip", "/Root/orders") + reader_metrics.record_message_processing_duration(0.04, "gzip", "/Root/orders") + reader_metrics.record_compression_duration(0.02, "gzip", "/Root/orders", error) + reader_metrics.record_message_processing_duration(0.06, "gzip", "/Root/orders", error) + + expected_base = { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/orders", + "codec": "gzip", + } + compression = { + point.attributes["outcome"]: point for point in _points(metrics_setup, TOPIC_READER_COMPRESSION_DURATION) + } + processing = { + point.attributes["outcome"]: point for point in _points(metrics_setup, TOPIC_READER_MESSAGE_PROCESSING_DURATION) + } + + assert (compression["ok"].count, compression["ok"].sum, compression["ok"].attributes) == ( + 1, + pytest.approx(0.01), + dict(expected_base, outcome="ok"), + ) + assert (processing["ok"].count, processing["ok"].sum, processing["ok"].attributes) == ( + 1, + pytest.approx(0.04), + dict(expected_base, outcome="ok"), + ) + assert (compression["error"].count, compression["error"].sum, compression["error"].attributes) == ( + 1, + pytest.approx(0.02), + dict(expected_base, outcome="error", **{"error.type": "ValueError"}), + ) + assert (processing["error"].count, processing["error"].sum, processing["error"].attributes) == ( + 1, + pytest.approx(0.06), + dict(expected_base, outcome="error", **{"error.type": "ValueError"}), + ) + + +def test_topic_reader_commit_queued_messages_are_split_by_topic(metrics_setup): + from tests.observability.conftest import FakeDriverConfig + from ydb.observability.metrics import ( + TOPIC_READER_COMMIT_QUEUED, + create_topic_reader_metrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + reader_metrics = create_topic_reader_metrics( + FakeDriver(), + consumer_name="analytics", + reader_name="payments-worker", + ) + reader_metrics.record_commit_queued(2, "/Root/orders") + reader_metrics.record_commit_queued(3, "/Root/refunds") + reader_metrics.record_commit_queued(1, "/Root/orders") + reader_metrics.record_commit_queued(0, "/Root/ignored") + + points = _points(metrics_setup, TOPIC_READER_COMMIT_QUEUED) + values = {point.attributes["topic"]: (point.value, point.attributes) for point in points} + assert values == { + "/Root/orders": ( + 3, + { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/orders", + }, + ), + "/Root/refunds": ( + 3, + { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/refunds", + }, + ), + } + + +def test_topic_reader_commit_acknowledged_messages_are_split_by_topic(metrics_setup): + from tests.observability.conftest import FakeDriverConfig + from ydb.observability.metrics import ( + TOPIC_READER_COMMIT_ACKNOWLEDGED, + create_topic_reader_metrics, + ) + + class FakeDriver: + _driver_config = FakeDriverConfig( + endpoint="grpc://localhost:2136", + database="/Root", + ) + + reader_metrics = create_topic_reader_metrics( + FakeDriver(), + consumer_name="analytics", + reader_name="payments-worker", + ) + reader_metrics.record_commit_acknowledged(2, "/Root/orders") + reader_metrics.record_commit_acknowledged(3, "/Root/refunds") + reader_metrics.record_commit_acknowledged(1, "/Root/orders") + reader_metrics.record_commit_acknowledged(0, "/Root/ignored") + + points = _points(metrics_setup, TOPIC_READER_COMMIT_ACKNOWLEDGED) + values = {point.attributes["topic"]: (point.value, point.attributes) for point in points} + assert values == { + "/Root/orders": ( + 3, + { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/orders", + }, + ), + "/Root/refunds": ( + 3, + { + "endpoint": "localhost:2136", + "database": "/Root", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/refunds", + }, + ), + } + + +def test_topic_reader_commit_offset_lag_tracks_maximum_across_partitions(metrics_setup): + from ydb.observability.metrics import ( + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + create_topic_reader_metrics, + ) + + reader_metrics = create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name="payments-worker", + ) + first = _topic_reader_partition_session("/Root/orders", 1, 10, committed_offset=10) + second = _topic_reader_partition_session("/Root/orders", 2, 10, committed_offset=100) + other = _topic_reader_partition_session("/Root/refunds", 3, 10, committed_offset=20) + try: + reader_metrics.add_partition_session(first.topic_path, first) + reader_metrics.add_partition_session(second.topic_path, second) + reader_metrics.add_partition_session(other.topic_path, other) + + assert { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) + } == {"/Root/orders": 0, "/Root/refunds": 0} + + reader_metrics.record_commit_offset_queued(first, 15) + reader_metrics.record_commit_offset_queued(second, 108) + reader_metrics.record_commit_offset_queued(other, 23) + assert { + point.attributes["topic"]: (point.value, point.attributes) + for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) + } == { + "/Root/orders": ( + 8, + { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/orders", + }, + ), + "/Root/refunds": ( + 3, + { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + "topic": "/Root/refunds", + }, + ), + } + + reader_metrics.record_commit_offset_acknowledged(second, 108) + assert { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) + } == {"/Root/orders": 5, "/Root/refunds": 3} + + reader_metrics.record_commit_offset_acknowledged(first, 9) + assert { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) + } == {"/Root/orders": 5, "/Root/refunds": 3} + + reader_metrics.record_commit_offset_acknowledged(first, 15) + reader_metrics.record_commit_offset_acknowledged(other, 23) + assert { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) + } == {"/Root/orders": 0, "/Root/refunds": 0} + + reader_metrics.remove_partition_session(first.topic_path, first) + reader_metrics.remove_partition_session(second.topic_path, second) + assert { + point.attributes["topic"]: point.value + for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) + } == {"/Root/refunds": 0} + + reader_metrics.reset_partition_sessions() + assert _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) == [] + finally: + reader_metrics.close() + + +def test_topic_reader_session_errors_are_split_by_retry_decision(metrics_setup): + from ydb.observability.metrics import ( + TOPIC_READER_SESSION_ERRORS, + create_topic_reader_metrics, + ) + + reader_metrics = create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name="payments-worker", + ) + reader_metrics.record_session_error(issues.Unavailable("retry"), True) + reader_metrics.record_session_error(issues.Unavailable("retry"), True) + reader_metrics.record_session_error(issues.BadRequest("stop"), False) + + points = _points(metrics_setup, TOPIC_READER_SESSION_ERRORS) + assert {point.attributes["retry_decision"]: (point.value, point.attributes) for point in points} == { + "retry": ( + 2, + { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + "retry_decision": "retry", + "status_code": "UNAVAILABLE", + "error.type": "ydb_error", + }, + ), + "stop": ( + 1, + { + "endpoint": "", + "database": "", + "consumer": "analytics", + "reader.name": "payments-worker", + "retry_decision": "stop", + "status_code": "BAD_REQUEST", + "error.type": "ydb_error", + }, + ), + } + + +@pytest.mark.parametrize( + ("error", "status_code", "error_type"), + [ + (issues.ConnectionLost("lost"), "CONNECTION_LOST", "transport_error"), + (issues.ConnectionError("connection"), "ConnectionError", "transport_error"), + (issues.BadRequest("bad request"), "BAD_REQUEST", "ydb_error"), + (RuntimeError("unexpected"), "RuntimeError", "RuntimeError"), + ], +) +def test_topic_reader_session_errors_classify_failure(metrics_setup, error, status_code, error_type): + from ydb.observability.metrics import ( + TOPIC_READER_SESSION_ERRORS, + create_topic_reader_metrics, + ) + + reader_metrics = create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name="payments-worker", + ) + reader_metrics.record_session_error(error, True) + + point = _single_point(metrics_setup, TOPIC_READER_SESSION_ERRORS) + assert point.attributes["status_code"] == status_code + assert point.attributes["error.type"] == error_type + + +def test_topic_reader_credit_balance_tracks_stream_credit(metrics_setup): + from ydb.observability.metrics import ( + TOPIC_READER_CREDIT_BALANCE_BYTES, + create_topic_reader_metrics, + ) + + reader_metrics = create_topic_reader_metrics( + object(), + consumer_name="analytics", + reader_name="payments-worker", + ) + try: + assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 0 + + reader_metrics.reset_credit_balance_bytes(100) + assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 100 + + reader_metrics.add_credit_balance_bytes(-12) + assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 88 + + reader_metrics.add_credit_balance_bytes(4) + assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 92 + + reader_metrics.reset_credit_balance_bytes() + assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 0 + finally: + reader_metrics.close() + + assert _points(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES) == [] + + def test_topic_reader_local_buffer_messages_tracks_current_reader_buffer(metrics_setup): from tests.observability.conftest import FakeDriverConfig from ydb.observability.metrics import ( @@ -1652,6 +2068,7 @@ async def test_topic_reader_gauges_reset_across_stream_reconnect(metrics_setup, from ydb._topic_reader.topic_reader import PublicReaderSettings from ydb._topic_reader.topic_reader_asyncio import PublicAsyncIOReader, ReaderStream from ydb.observability.metrics import ( + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, TOPIC_READER_PARTITION_SESSION_COUNT, @@ -1695,6 +2112,7 @@ async def create_stream(reader_reconnector_id, driver, stream_settings, *, reade ) streams.append(stream) if len(streams) == 1: + metrics.reset_credit_balance_bytes(stream_settings.buffer_size_bytes) metrics.add_partition_session("/Root/orders") stream._on_read_response(_topic_read_response([(1, 2)], bytes_size=20)) await _decode_pending_topic_batches(stream) @@ -1702,6 +2120,7 @@ async def create_stream(reader_reconnector_id, driver, stream_settings, *, reade else: second_stream_create_started.set() await allow_second_stream.wait() + metrics.reset_credit_balance_bytes(stream_settings.buffer_size_bytes) metrics.add_partition_session("/Root/orders") stream._on_read_response(_topic_read_response([(1, 1)], bytes_size=10)) await _decode_pending_topic_batches(stream) @@ -1716,12 +2135,16 @@ async def create_stream(reader_reconnector_id, driver, stream_settings, *, reade first_age = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value assert 5.0 <= first_age < 10.0 assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1 + assert ( + _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == settings.buffer_size_bytes - 20 + ) streams[0]._set_first_error(issues.Unavailable("reconnect")) await asyncio.wait_for(second_stream_create_started.wait(), timeout=2) assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0 assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 0 assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 0 + assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 0 allow_second_stream.set() await asyncio.wait_for(second_stream_ready.wait(), timeout=1) @@ -1735,6 +2158,9 @@ async def wait_until_second_stream_is_installed(): second_age = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value assert 5.0 <= second_age < 10.0 assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1 + assert ( + _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == settings.buffer_size_bytes - 10 + ) finally: allow_second_stream.set() await reader.close(flush=False) @@ -1742,10 +2168,13 @@ async def wait_until_second_stream_is_installed(): assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == [] assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX) == [] assert _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) == [] + assert _points(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES) == [] def test_topic_reader_gauges_aggregate_duplicate_reader_attributes(metrics_setup, monkeypatch): from ydb.observability.metrics import ( + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, TOPIC_READER_PARTITION_SESSION_COUNT, @@ -1754,30 +2183,47 @@ def test_topic_reader_gauges_aggregate_duplicate_reader_attributes(metrics_setup first = create_topic_reader_metrics(object(), consumer_name=None, reader_name="shared") second = create_topic_reader_metrics(object(), consumer_name=None, reader_name="shared") + first_partition = _topic_reader_partition_session("/Root/events", 1, 1) + second_partition = _topic_reader_partition_session("/Root/events", 2, 1) + third_partition = _topic_reader_partition_session("/Root/events", 3, 2) try: first.record_received_messages([_topic_reader_batch(2, "/Root/events")], received_at=10.0) second.record_received_messages([_topic_reader_batch(3, "/Root/events")], received_at=20.0) - first.add_partition_session("/Root/events") - first.add_partition_session("/Root/events") - second.add_partition_session("/Root/events") + first.reset_credit_balance_bytes(100) + second.reset_credit_balance_bytes(50) + first.add_partition_session("/Root/events", first_partition) + first.add_partition_session("/Root/events", second_partition) + second.add_partition_session("/Root/events", third_partition) + first.record_commit_offset_queued(first_partition, 5) + first.record_commit_offset_queued(second_partition, 2) + second.record_commit_offset_queued(third_partition, 8) monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=30.0)) assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 5 assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0 assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 3 + assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 150 + assert _single_point(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX).value == 8 finally: first.close() second.close() -def test_topic_reader_received_metrics_omit_consumer(metrics_setup): +def test_topic_reader_metrics_omit_consumer(metrics_setup): from ydb.observability.metrics import ( + TOPIC_READER_COMMIT_ACKNOWLEDGED, + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_COMMIT_QUEUED, + TOPIC_READER_COMPRESSION_DURATION, + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_DELIVERED_MESSAGES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_MESSAGE_PROCESSING_DURATION, TOPIC_READER_PARTITION_SESSION_COUNT, TOPIC_READER_RECEIVED_BYTES, TOPIC_READER_RECEIVED_MESSAGES, + TOPIC_READER_SESSION_ERRORS, create_topic_reader_metrics, ) @@ -1790,7 +2236,15 @@ def test_topic_reader_received_metrics_omit_consumer(metrics_setup): reader_metrics.record_received_messages([_topic_reader_batch(1, "/Root/events")]) reader_metrics.record_received_bytes(10) reader_metrics.record_delivered_messages(1, "/Root/events") - reader_metrics.add_partition_session("/Root/events") + reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events") + reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events") + reader_metrics.record_commit_queued(1, "/Root/events") + reader_metrics.record_commit_acknowledged(1, "/Root/events") + reader_metrics.record_session_error(issues.BadRequest("stop"), False) + reader_metrics.reset_credit_balance_bytes(100) + partition_session = _topic_reader_partition_session("/Root/events", 1, 1) + reader_metrics.add_partition_session("/Root/events", partition_session) + reader_metrics.record_commit_offset_queued(partition_session, 1) reader_attributes = { "endpoint": "", @@ -1800,6 +2254,19 @@ def test_topic_reader_received_metrics_omit_consumer(metrics_setup): message_attributes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES).attributes bytes_attributes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES).attributes delivered_attributes = _single_point(metrics_setup, TOPIC_READER_DELIVERED_MESSAGES).attributes + compression_attributes = _single_point(metrics_setup, TOPIC_READER_COMPRESSION_DURATION).attributes + processing_attributes = _single_point(metrics_setup, TOPIC_READER_MESSAGE_PROCESSING_DURATION).attributes + commit_queued_attributes = _single_point(metrics_setup, TOPIC_READER_COMMIT_QUEUED).attributes + commit_acknowledged_attributes = _single_point( + metrics_setup, + TOPIC_READER_COMMIT_ACKNOWLEDGED, + ).attributes + commit_offset_lag_attributes = _single_point( + metrics_setup, + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + ).attributes + session_error_attributes = _single_point(metrics_setup, TOPIC_READER_SESSION_ERRORS).attributes + credit_attributes = _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).attributes buffer_attributes = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).attributes buffer_age_attributes = _single_point( metrics_setup, @@ -1812,6 +2279,19 @@ def test_topic_reader_received_metrics_omit_consumer(metrics_setup): assert message_attributes == dict(reader_attributes, topic="/Root/events") assert bytes_attributes == reader_attributes assert delivered_attributes == dict(reader_attributes, topic="/Root/events") + duration_attributes = dict(reader_attributes, topic="/Root/events", codec="gzip", outcome="ok") + assert compression_attributes == duration_attributes + assert processing_attributes == duration_attributes + assert commit_queued_attributes == dict(reader_attributes, topic="/Root/events") + assert commit_acknowledged_attributes == dict(reader_attributes, topic="/Root/events") + assert commit_offset_lag_attributes == dict(reader_attributes, topic="/Root/events") + assert session_error_attributes == dict( + reader_attributes, + retry_decision="stop", + status_code="BAD_REQUEST", + **{"error.type": "ydb_error"}, + ) + assert credit_attributes == reader_attributes assert buffer_attributes == reader_attributes assert buffer_age_attributes == reader_attributes assert partition_session_attributes == dict(reader_attributes, topic="/Root/events") @@ -1820,14 +2300,21 @@ def test_topic_reader_received_metrics_omit_consumer(metrics_setup): @pytest.mark.parametrize("reader_name", [None, ""]) -def test_topic_reader_received_metrics_require_explicit_reader_name(metrics_setup, reader_name): +def test_topic_reader_metrics_require_explicit_reader_name(metrics_setup, reader_name): from ydb.observability.metrics import ( + TOPIC_READER_COMMIT_ACKNOWLEDGED, + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_COMMIT_QUEUED, + TOPIC_READER_COMPRESSION_DURATION, + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_DELIVERED_MESSAGES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_MESSAGE_PROCESSING_DURATION, TOPIC_READER_PARTITION_SESSION_COUNT, TOPIC_READER_RECEIVED_BYTES, TOPIC_READER_RECEIVED_MESSAGES, + TOPIC_READER_SESSION_ERRORS, create_topic_reader_metrics, ) @@ -1839,13 +2326,26 @@ def test_topic_reader_received_metrics_require_explicit_reader_name(metrics_setu reader_metrics.record_received_messages([_topic_reader_batch(1, "/Root/events")]) reader_metrics.record_received_bytes(10) reader_metrics.record_delivered_messages(1, "/Root/events") + reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events") + reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events") + reader_metrics.record_commit_queued(1, "/Root/events") + reader_metrics.record_commit_acknowledged(1, "/Root/events") + reader_metrics.record_session_error(issues.Unavailable("retry"), True) + reader_metrics.reset_credit_balance_bytes(100) reader_metrics.add_partition_session("/Root/events") metrics = _metrics_by_name(metrics_setup) assert TOPIC_READER_RECEIVED_MESSAGES not in metrics assert TOPIC_READER_RECEIVED_BYTES not in metrics + assert TOPIC_READER_COMMIT_ACKNOWLEDGED not in metrics + assert TOPIC_READER_COMMIT_OFFSET_LAG_MAX not in metrics + assert TOPIC_READER_COMMIT_QUEUED not in metrics + assert TOPIC_READER_COMPRESSION_DURATION not in metrics + assert TOPIC_READER_SESSION_ERRORS not in metrics + assert TOPIC_READER_CREDIT_BALANCE_BYTES not in metrics assert TOPIC_READER_DELIVERED_MESSAGES not in metrics assert TOPIC_READER_LOCAL_BUFFER_MESSAGES not in metrics + assert TOPIC_READER_MESSAGE_PROCESSING_DURATION not in metrics assert TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX not in metrics assert TOPIC_READER_PARTITION_SESSION_COUNT not in metrics @@ -2510,3 +3010,266 @@ def fake_span_ctx(**kwargs): assert qs._session_metrics._counted assert _single_point_for_pool(metrics_setup, QUERY_SESSION_COUNT, "async-session-pool").value == 1 qs._close_session() + + +def test_opentelemetry_load_tank_topic_payload_has_exact_size(): + from examples.opentelemetry.load_tank import _topic_payload + + assert _topic_payload(42, 64).startswith(b"42:") + assert len(_topic_payload(42, 64)) == 64 + assert len(_topic_payload(42, 1)) == 1 + + +@pytest.mark.asyncio +async def test_opentelemetry_load_tank_retry_probe_records_multiple_attempts(metrics_setup, monkeypatch): + import asyncio + from types import SimpleNamespace + + import ydb + from examples.opentelemetry.load_tank import _retry_probe_worker + from ydb.observability.metrics import RETRY_ATTEMPTS + + stop = asyncio.Event() + retry_operation_async = ydb.retry_operation_async + + async def run_once(*args, **kwargs): + result = await retry_operation_async(*args, **kwargs) + stop.set() + return result + + monkeypatch.setattr(ydb, "retry_operation_async", run_once) + await _retry_probe_worker( + SimpleNamespace(retry_probe_interval=1.0, retry_probe_failures=2), + stop, + ) + + retry_attempts = _single_point(metrics_setup, RETRY_ATTEMPTS) + assert retry_attempts.count == 1 + assert retry_attempts.sum == 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("query_error", "should_raise"), + [ + (issues.SchemeError("expected query failure"), False), + (RuntimeError("unexpected failure"), True), + ], +) +async def test_opentelemetry_load_tank_error_worker_only_suppresses_expected_error( + query_error, + should_raise, +): + from examples.opentelemetry.load_tank import _error_worker + + stop = asyncio.Event() + + async def execute_with_retries(_): + stop.set() + raise query_error + + pool = SimpleNamespace(execute_with_retries=execute_with_retries) + config = SimpleNamespace(error_query="missing table", error_rps=1) + if should_raise: + with pytest.raises(RuntimeError, match="unexpected failure"): + await _error_worker(pool, config, stop) + else: + await _error_worker(pool, config, stop) + + +def test_opentelemetry_metric_verifier_detects_prometheus_series(monkeypatch): + from examples.opentelemetry.verify_metrics import _prometheus_has_series + + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + def read(self): + return b'{"status":"success","data":{"result":[{"value":[1,"1"]}]}}' + + urlopen = MagicMock(return_value=FakeResponse()) + monkeypatch.setattr("urllib.request.urlopen", urlopen) + + assert _prometheus_has_series("http://prometheus:9090", "ydb_query_session_count") + assert "query=count%28ydb_query_session_count%29" in urlopen.call_args.args[0] + + +def test_opentelemetry_metric_verifier_scopes_every_check_to_current_run(): + from examples.opentelemetry.verify_metrics import _metric_checks + + checks = _metric_checks('current-run"with\\escaping') + + assert all('service_instance_id="current-run\\"with\\\\escaping"' in check.promql for check in checks) + error_check = next(check for check in checks if check.requires_error_probe) + assert 'reader_name="load-tank-topic-error-probe"' in error_check.promql + assert 'retry_decision="stop"' in error_check.promql + assert 'status_code="SCHEME_ERROR"' in error_check.promql + + +def test_opentelemetry_metric_verifier_respects_total_timeout(monkeypatch): + from examples.opentelemetry import verify_metrics + + now = 100.0 + request_timeouts = [] + + def monotonic(): + return now + + def query_has_result(_, __, timeout): + nonlocal now + request_timeouts.append(timeout) + now += timeout + return False + + monkeypatch.setattr(verify_metrics.time, "monotonic", monotonic) + monkeypatch.setattr(verify_metrics, "_prometheus_query_has_result", query_has_result) + + checks = ( + verify_metrics.MetricCheck("first", "first_metric"), + verify_metrics.MetricCheck("second", "second_metric"), + ) + missing = verify_metrics.wait_for_metrics( + "http://prometheus:9090", + checks, + timeout=3, + poll_interval=10, + ) + + assert request_timeouts == [3] + assert tuple(missing) == ("first_metric", "second_metric") + + +def test_opentelemetry_dashboard_keeps_topic_reader_instances_separate(): + import json + from pathlib import Path + + dashboard = json.loads( + Path("examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json").read_text(encoding="utf-8") + ) + topic_reader_targets = [ + target + for panel in dashboard["panels"] + for target in panel.get("targets", ()) + if "ydb_topic_reader_" in target.get("expr", "") + ] + + assert topic_reader_targets + for target in topic_reader_targets: + assert all( + label in target["expr"] + for label in ("service_instance_id", "endpoint", "database", "consumer") + ) + + +def test_opentelemetry_load_tank_publishes_current_run_id(): + from pathlib import Path + from tempfile import TemporaryDirectory + + from examples.opentelemetry.load_tank import _publish_run_id + + with TemporaryDirectory(dir=".") as directory: + run_id_file = Path(directory) / "metrics" / "run-id" + ready_file = Path(directory) / "ready" + _publish_run_id("current-run", str(run_id_file), str(ready_file)) + + assert run_id_file.read_text(encoding="utf-8") == "current-run\n" + assert ready_file.is_file() + + +@pytest.mark.asyncio +async def test_opentelemetry_load_tank_propagates_background_failure(): + from examples.opentelemetry.load_tank import _await_with_background + + async def fail(): + raise ValueError("topic failed") + + task = asyncio.create_task(fail()) + with pytest.raises(RuntimeError, match="topic stream worker failed") as error: + await _await_with_background(asyncio.sleep(60), {task: "topic stream worker"}) + + assert isinstance(error.value.__cause__, ValueError) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancelled", [False, True]) +async def test_opentelemetry_load_tank_detects_stopped_background_task(cancelled): + from examples.opentelemetry.load_tank import _await_with_background + + task = asyncio.create_task(asyncio.sleep(0 if not cancelled else 60)) + if cancelled: + task.cancel() + + message = "worker was cancelled unexpectedly" if cancelled else "worker stopped unexpectedly" + with pytest.raises(RuntimeError, match=message): + await _await_with_background(asyncio.sleep(60), {task: "worker"}) + + +@pytest.mark.asyncio +async def test_opentelemetry_load_tank_propagates_failure_during_worker_cancellation(): + from examples.opentelemetry.load_tank import _cancel_background_tasks + + async def fail_during_cancellation(): + try: + await asyncio.sleep(60) + except asyncio.CancelledError as error: + raise ValueError("cleanup failed") from error + + task = asyncio.create_task(fail_during_cancellation()) + await asyncio.sleep(0) + with pytest.raises(RuntimeError, match="worker failed") as error: + await _cancel_background_tasks({task: "worker"}, raise_failures=True) + + assert isinstance(error.value.__cause__, ValueError) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("probe_error", "expected_error"), + [ + (issues.SchemeError("missing topic"), None), + (RuntimeError("unexpected failure"), RuntimeError), + ], +) +async def test_opentelemetry_load_tank_topic_error_probe_is_strict( + monkeypatch, + probe_error, + expected_error, +): + import ydb + from examples.opentelemetry.load_tank import _topic_error_probe + + class Reader: + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + async def wait_message(self): + raise probe_error + + class Driver: + def __init__(self, **_): + self.topic_client = SimpleNamespace(reader=lambda *args, **kwargs: Reader()) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + pass + + monkeypatch.setattr(ydb.aio, "Driver", Driver) + config = SimpleNamespace( + database="/local", + topic_error_probe_endpoint="grpc://localhost:2136", + topic_error_probe_seconds=1, + ) + + if expected_error is None: + await _topic_error_probe(config) + else: + with pytest.raises(expected_error, match="unexpected failure"): + await _topic_error_probe(config) diff --git a/ydb/_topic_reader/datatypes.py b/ydb/_topic_reader/datatypes.py index 19f706e29..03d255042 100644 --- a/ydb/_topic_reader/datatypes.py +++ b/ydb/_topic_reader/datatypes.py @@ -94,22 +94,26 @@ def add_waiter(self, end_offset: int) -> "PartitionSession.CommitAckWaiter": return waiter - def ack_notify(self, offset: int): + def ack_notify(self, offset: int) -> int: self._ensure_not_closed() - self.committed_offset = offset + previous_committed_offset = self.committed_offset + self.committed_offset = max(previous_committed_offset, offset) + acknowledged_messages = self.committed_offset - previous_committed_offset if not self._ack_waiters: # todo log warning # must be never receive ack for not sended request - return + return acknowledged_messages while self._ack_waiters: - if self._ack_waiters[0].end_offset > offset: + if self._ack_waiters[0].end_offset > self.committed_offset: break waiter = self._ack_waiters.popleft() waiter._finish_ok() + return acknowledged_messages + def _update_last_commited_offset_if_needed(self, offset: int): self.committed_offset = max(self.committed_offset, offset) @@ -168,6 +172,7 @@ class PublicBatch(ICommittable, ISessionAlive): _partition_session: PartitionSession _bytes_size: int _codec: Codec + _received_at: Optional[float] = field(default=None, compare=False, repr=False) def _commit_get_partition_session(self) -> PartitionSession: return self.messages[0]._commit_get_partition_session() diff --git a/ydb/_topic_reader/datatypes_test.py b/ydb/_topic_reader/datatypes_test.py index 19a8bf36b..25571127a 100644 --- a/ydb/_topic_reader/datatypes_test.py +++ b/ydb/_topic_reader/datatypes_test.py @@ -34,7 +34,7 @@ def session(self) -> PartitionSession: ([1], 10, [1], []), ([1, 2, 3], 10, [1, 2, 3], []), ([1, 2, 10, 20], 10, [1, 2, 10], [20]), - ([10, 20], 1, [], [10, 20]), + ([10, 20], 1, [10], [20]), ], ) async def test_ack_notify( @@ -66,7 +66,34 @@ def add_notify(future, notified_offset): notified.sort() assert notified == offsets_notified - assert session.committed_offset == notify_offset + assert session.committed_offset == max(self.session_comitted_offset, notify_offset) + + async def test_ack_notify_returns_monotonic_committed_offset_advance(self, session): + first = session.add_waiter(session.committed_offset + 2) + second = session.add_waiter(session.committed_offset + 5) + + assert session.ack_notify(session.committed_offset + 3) == 3 + assert first.future.done() + assert not second.future.done() + + assert session.ack_notify(session.committed_offset + 2) == 2 + assert second.future.done() + assert session.ack_notify(session.committed_offset) == 0 + + stale_offset = session.committed_offset - 1 + assert session.ack_notify(stale_offset) == 0 + assert session.committed_offset == stale_offset + 1 + + async def test_ack_notify_does_not_double_count_overlapping_ranges(self, session): + first = session.add_waiter(session.committed_offset + 5) + second = session.add_waiter(session.committed_offset + 7) + + assert session.ack_notify(session.committed_offset + 7) == 7 + assert first.future.done() + assert second.future.done() + + async def test_ack_notify_counts_offset_advance_without_waiters(self, session): + assert session.ack_notify(session.committed_offset + 2) == 2 # noinspection PyTypeChecker @pytest.mark.parametrize( diff --git a/ydb/_topic_reader/topic_reader_asyncio.py b/ydb/_topic_reader/topic_reader_asyncio.py index 8d6371be8..5c1a7aa38 100644 --- a/ydb/_topic_reader/topic_reader_asyncio.py +++ b/ydb/_topic_reader/topic_reader_asyncio.py @@ -4,6 +4,7 @@ import concurrent.futures import gzip import math +from time import monotonic import typing from asyncio import Task from collections import defaultdict, OrderedDict @@ -48,6 +49,18 @@ logger = logging.getLogger(__name__) +_CODEC_NAMES = { + Codec.CODEC_UNSPECIFIED: "unspecified", + Codec.CODEC_RAW: "raw", + Codec.CODEC_GZIP: "gzip", + Codec.CODEC_LZOP: "lzop", + Codec.CODEC_ZSTD: "zstd", +} + + +def _codec_name(codec: int) -> str: + return _CODEC_NAMES.get(codec, str(codec)) + class TopicReaderError(YdbError): pass @@ -314,7 +327,10 @@ async def _connection_loop(self): await self._stream_reader.wait_error() except BaseException as err: logger.debug("%s connection attempt=%s failed: %s", self._log_prefix, attempt, err) + if self._closed: + return retry_info = check_retriable_error(err, self._settings._retry_settings(), attempt) + self._metrics.record_session_error(err, retry_info.is_retriable) if not retry_info.is_retriable: logger.debug("%s stop connection loop: %s", self._log_prefix, err) self._set_first_error(err) @@ -464,25 +480,31 @@ async def _commit_batches_with_tx(self, tx: "BaseQueryTxContext"): for topic_path in grouped_batches: topic_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets(path=topic_path, partitions=[]) for partition_id in grouped_batches[topic_path]: + ranges = [batch._commit_get_offsets_range() for batch in grouped_batches[topic_path][partition_id]] partition_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets( partition_id=partition_id, - partition_offsets=[ - batch._commit_get_offsets_range() for batch in grouped_batches[topic_path][partition_id] - ], + partition_offsets=ranges, ) topic_offsets.partitions.append(partition_offsets) request.topics.append(topic_offsets) + for topic_offsets in request.topics: + for partition_offsets in topic_offsets.partitions: + for offsets in partition_offsets.partition_offsets: + self._metrics.record_commit_queued(offsets.end - offsets.start, topic_offsets.path) + try: - return await self._do_commit_batches_with_tx_call(request) + result = await self._do_commit_batches_with_tx_call(request) except BaseException: err = issues.ClientInternalError("Failed to update offsets in tx.") tx._set_external_error(err) if self._stream_reader is not None: self._stream_reader._set_first_error(err) - finally: if tx_id in self._tx_to_batches_map: del self._tx_to_batches_map[tx_id] + return None + + return result async def _do_commit_batches_with_tx_call(self, request: UpdateOffsetsInTransactionRequest): args = [ @@ -509,8 +531,15 @@ async def _handle_after_tx_rollback(self, tx: "BaseQueryTxContext", exc: Optiona async def _handle_after_tx_commit(self, tx: "BaseQueryTxContext", exc: Optional[BaseException]) -> None: tx_id = tx.tx_id - if tx_id is not None and tx_id in self._tx_to_batches_map: - del self._tx_to_batches_map[tx_id] + batches = self._tx_to_batches_map.pop(tx_id, []) if tx_id is not None else [] + + if exc is None: + for batch in batches: + offsets = batch._commit_get_offsets_range() + self._metrics.record_commit_acknowledged( + offsets.end - offsets.start, + batch._partition_session.topic_path, + ) if exc is not None and self._stream_reader is not None: self._stream_reader._set_first_error( @@ -826,6 +855,11 @@ def commit(self, batch: datatypes.ICommittable) -> datatypes.PartitionSession.Co ) if self._stream is not None: self._stream.write(StreamReadMessage.FromClient(client_message=client_message)) + self._metrics.record_commit_offset_queued(partition_session, commit_range.end) + self._metrics.record_commit_queued( + commit_range.end - commit_range.start, + partition_session.topic_path, + ) return waiter @@ -843,13 +877,15 @@ async def _handle_background_errors(self): async def _read_messages_loop(self): try: logger.debug("%s start read loop", self._log_prefix) + initial_credit_bytes = self._buffer_size_bytes self._stream.write( StreamReadMessage.FromClient( client_message=StreamReadMessage.ReadRequest( - bytes_size=self._buffer_size_bytes, + bytes_size=initial_credit_bytes, ), ) ) + self._metrics.reset_credit_balance_bytes(initial_credit_bytes) while True: try: message = await self._stream.receive() # type: StreamReadMessage.FromServer @@ -952,7 +988,7 @@ async def _on_start_partition_session(self, message: StreamReadMessage.StartPart reader_stream_id=self._id, ) self._partition_sessions[partition_session.id] = partition_session - self._metrics.add_partition_session(partition_session.topic_path) + self._metrics.add_partition_session(partition_session.topic_path, partition_session) read_offset = None @@ -986,7 +1022,7 @@ def _on_partition_session_stop(self, message: StreamReadMessage.StopPartitionSes partition = self._partition_sessions.pop(message.partition_session_id) partition.close() - self._metrics.remove_partition_session(partition.topic_path) + self._metrics.remove_partition_session(partition.topic_path, partition) if message.graceful and self._stream is not None: self._stream.write( @@ -1014,6 +1050,7 @@ def _on_read_response(self, message: StreamReadMessage.ReadResponse): batches = self._read_response_to_batches(message) for batch in batches: + batch._received_at = received_at self._batches_to_decode.put_nowait(batch) self._metrics.record_received_messages(batches, received_at=received_at) @@ -1024,23 +1061,34 @@ def _on_commit_response(self, message: StreamReadMessage.CommitOffsetResponse): continue session = self._partition_sessions[partition_offset.partition_session_id] - session.ack_notify(partition_offset.committed_offset) + acknowledged_messages = session.ack_notify(partition_offset.committed_offset) + self._metrics.record_commit_offset_acknowledged( + session, + partition_offset.committed_offset, + ) + self._metrics.record_commit_acknowledged( + acknowledged_messages, + session.topic_path, + ) def _buffer_consume_bytes(self, bytes_size): self._buffer_size_bytes -= bytes_size + self._metrics.add_credit_balance_bytes(-bytes_size) def _buffer_release_bytes(self, bytes_size): self._pending_buffer_release_bytes += bytes_size if self._pending_buffer_release_bytes >= self._min_buffer_release_bytes: - self._buffer_size_bytes += self._pending_buffer_release_bytes + released_bytes = self._pending_buffer_release_bytes self._stream.write( StreamReadMessage.FromClient( client_message=StreamReadMessage.ReadRequest( - bytes_size=self._pending_buffer_release_bytes, + bytes_size=released_bytes, ) ) ) + self._buffer_size_bytes += released_bytes self._pending_buffer_release_bytes = 0 + self._metrics.add_credit_balance_bytes(released_bytes) def _read_response_to_batches(self, message: StreamReadMessage.ReadResponse) -> typing.List[datatypes.PublicBatch]: batches: typing.List[datatypes.PublicBatch] = [] @@ -1121,19 +1169,66 @@ async def _decode_batch_inplace(self, batch): try: decode_func = self._decoders[batch._codec] except KeyError: - raise PublicTopicReaderUnexpectedCodecError("Receive message with unexpected codec: %s" % batch._codec) + error = PublicTopicReaderUnexpectedCodecError("Receive message with unexpected codec: %s" % batch._codec) + if batch._received_at is not None: + processing_duration = monotonic() - batch._received_at + codec = _codec_name(batch._codec) + for _ in batch.messages: + self._metrics.record_compression_duration(0.0, codec, batch._partition_session.topic_path, error) + self._metrics.record_message_processing_duration( + processing_duration, + codec, + batch._partition_session.topic_path, + error, + ) + raise error decode_data_futures = [] + if batch._received_at is None: + for message in batch.messages: + future = self._loop.run_in_executor(self._decode_executor, decode_func, message.data) + decode_data_futures.append(future) + + decoded_data = await asyncio.gather(*decode_data_futures) + for index, message in enumerate(batch.messages): + message.data = decoded_data[index] + + batch._codec = Codec.CODEC_RAW + return + for message in batch.messages: - future = self._loop.run_in_executor(self._decode_executor, decode_func, message.data) + future = self._loop.run_in_executor( + self._decode_executor, self._decode_with_timing, decode_func, message.data + ) decode_data_futures.append(future) decoded_data = await asyncio.gather(*decode_data_futures) + processing_duration = monotonic() - batch._received_at + processing_error = next((error for _, _, error in decoded_data if error is not None), None) + codec = _codec_name(batch._codec) + topic = batch._partition_session.topic_path + + for _, compression_duration, error in decoded_data: + self._metrics.record_compression_duration(compression_duration, codec, topic, error) + self._metrics.record_message_processing_duration(processing_duration, codec, topic, processing_error) + + if processing_error is not None: + raise processing_error + for index, message in enumerate(batch.messages): - message.data = decoded_data[index] + message.data = decoded_data[index][0] batch._codec = Codec.CODEC_RAW + @staticmethod + def _decode_with_timing(decode_func, data): + started_at = monotonic() + try: + decoded = decode_func(data) + return decoded, monotonic() - started_at, None + except Exception as error: + return None, monotonic() - started_at, error + def _set_first_error(self, err: YdbError): try: self._first_error.set_result(err) @@ -1183,7 +1278,10 @@ async def close(self, flush: bool): await asyncio.wait(self._background_tasks) finally: try: - self._metrics.reset_local_buffer_messages() + self._metrics.reset_credit_balance_bytes() finally: - self._metrics.reset_partition_sessions() + try: + self._metrics.reset_local_buffer_messages() + finally: + self._metrics.reset_partition_sessions() logger.debug("%s was closed", self._log_prefix) diff --git a/ydb/_topic_reader/topic_reader_asyncio_test.py b/ydb/_topic_reader/topic_reader_asyncio_test.py index 7de4e5db3..e6fb0be25 100644 --- a/ydb/_topic_reader/topic_reader_asyncio_test.py +++ b/ydb/_topic_reader/topic_reader_asyncio_test.py @@ -14,7 +14,12 @@ from . import datatypes, topic_reader_asyncio from .datatypes import PublicBatch, PublicMessage from .topic_reader import PublicReaderSettings -from .topic_reader_asyncio import ReaderStream, ReaderReconnector, TopicReaderError +from .topic_reader_asyncio import ( + ReaderStream, + ReaderReconnector, + TopicReaderError, + TopicReaderStreamClosedError, +) from .._grpc.grpcwrapper.common_utils import SupportedDriverType, ServerStatus from .._grpc.grpcwrapper.ydb_topic import ( StreamReadMessage, @@ -343,6 +348,7 @@ def assert_accepted_for_decoding(batches, *, received_at): assert len(batches) == 1 assert len(batches[0].messages) == 2 assert batches[0]._partition_session.topic_path == default_reader_settings.topic + assert batches[0]._received_at == 12.5 assert received_at == 12.5 metrics.record_received_bytes.side_effect = assert_bytes_recorded_on_receipt @@ -358,6 +364,7 @@ def assert_accepted_for_decoding(batches, *, received_at): ) metrics.record_received_messages.assert_called_once() metrics.record_received_bytes.assert_called_once_with(1) + metrics.add_credit_balance_bytes.assert_called_once_with(-1) first = reader.receive_batch_nowait(max_messages=1) second = reader.receive_batch_nowait(max_messages=1) @@ -370,6 +377,10 @@ def assert_accepted_for_decoding(batches, *, received_at): finally: await reader.close(False) + assert metrics.reset_credit_balance_bytes.call_args_list == [ + mock.call(default_reader_settings.buffer_size_bytes), + mock.call(), + ] metrics.reset_local_buffer_messages.assert_called_once_with() metrics.reset_partition_sessions.assert_called_once_with() @@ -392,6 +403,7 @@ async def test_received_bytes_are_recorded_before_response_conversion(self, defa reader._on_read_response(response) metrics.record_received_bytes.assert_called_once_with(5) + metrics.add_credit_balance_bytes.assert_called_once_with(-5) metrics.record_received_messages.assert_not_called() async def test_received_bytes_are_recorded_once_for_multitopic_response(self, default_reader_settings): @@ -476,6 +488,13 @@ class TestError(Exception): ), True, ), + ( + OffsetsRange( + partition_session_committed_offset, + partition_session_committed_offset + 3, + ), + True, + ), ( OffsetsRange( partition_session_committed_offset - 1, @@ -493,6 +512,9 @@ async def test_send_commit_messages( commit: OffsetsRange, send_range: bool, ): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + @dataclass class Committable(datatypes.ICommittable): start: int @@ -522,15 +544,38 @@ async def wait_message(): ] ) assert partition_session._ack_waiters[-1].end_offset == commit.end + metrics.record_commit_queued.assert_called_once_with( + commit.end - commit.start, + partition_session.topic_path, + ) + metrics.record_commit_offset_queued.assert_called_once_with(partition_session, commit.end) else: assert waiter.future.done() + metrics.record_commit_queued.assert_not_called() + metrics.record_commit_offset_queued.assert_not_called() with pytest.raises(WaitConditionError): msg = await wait_message() pass assert start_ack_waiters == partition_session._ack_waiters + async def test_failed_commit_write_is_not_recorded(self, stream_reader, stream, partition_session): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + message = self.create_message(partition_session, 1, 1) + + with mock.patch.object(stream, "write", side_effect=RuntimeError("write failed")): + with pytest.raises(RuntimeError, match="write failed"): + stream_reader.commit(message) + + metrics.record_commit_queued.assert_not_called() + metrics.record_commit_offset_queued.assert_not_called() + waiter = partition_session._ack_waiters.pop() + waiter.future.cancel() + async def test_commit_ack_received(self, stream_reader, stream, partition_session, second_partition_session): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics offset1 = self.partition_session_committed_offset + 1 waiter1 = partition_session.add_waiter(offset1) @@ -557,6 +602,39 @@ async def test_commit_ack_received(self, stream_reader, stream, partition_sessio await wait_for_fast(waiter1.future) await wait_for_fast(waiter2.future) + assert metrics.record_commit_acknowledged.call_args_list == [ + mock.call(1, partition_session.topic_path), + mock.call(2, second_partition_session.topic_path), + ] + assert metrics.record_commit_offset_acknowledged.call_args_list == [ + mock.call(partition_session, offset1), + mock.call(second_partition_session, offset2), + ] + + stream_reader._on_commit_response( + StreamReadMessage.CommitOffsetResponse( + partitions_committed_offsets=[ + StreamReadMessage.CommitOffsetResponse.PartitionCommittedOffset( + partition_session_id=partition_session.id, + committed_offset=offset1, + ), + StreamReadMessage.CommitOffsetResponse.PartitionCommittedOffset( + partition_session_id=999, + committed_offset=100, + ), + ] + ) + ) + assert metrics.record_commit_acknowledged.call_args_list == [ + mock.call(1, partition_session.topic_path), + mock.call(2, second_partition_session.topic_path), + mock.call(0, partition_session.topic_path), + ] + assert metrics.record_commit_offset_acknowledged.call_args_list == [ + mock.call(partition_session, offset1), + mock.call(second_partition_session, offset2), + mock.call(partition_session, offset1), + ] async def test_close_ack_waiters_when_close_stream_reader( self, stream_reader_started: ReaderStream, partition_session @@ -587,6 +665,7 @@ async def test_close_resets_local_buffer_after_flush_failure(self, default_reade assert reader._closed assert all(task.done() for task in reader._background_tasks) + metrics.reset_credit_balance_bytes.assert_called_once_with() metrics.reset_local_buffer_messages.assert_called_once_with() metrics.reset_partition_sessions.assert_called_once_with() @@ -776,6 +855,106 @@ async def test_decode_loop(self, stream_reader, batch: PublicBatch, data_out: ty assert batch == expected + async def test_decode_records_compression_and_processing_duration(self, stream_reader): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + partition_session = stub_partition_session() + partition_session.topic_path = "/Root/orders" + message = stub_message(1) + message._partition_session = partition_session + message.data = gzip.compress(b"payload") + batch = PublicBatch( + messages=[message], + _partition_session=partition_session, + _bytes_size=len(message.data), + _codec=Codec.CODEC_GZIP, + _received_at=10.0, + ) + + with mock.patch.object(topic_reader_asyncio, "monotonic", side_effect=[11.0, 11.25, 11.5]): + await stream_reader._decode_batch_inplace(batch) + + assert message.data == b"payload" + assert batch._codec == Codec.CODEC_RAW + metrics.record_compression_duration.assert_called_once_with(0.25, "gzip", "/Root/orders", None) + metrics.record_message_processing_duration.assert_called_once_with(1.5, "gzip", "/Root/orders", None) + + async def test_decode_failure_records_both_durations_and_is_rethrown(self, stream_reader): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + partition_session = stub_partition_session() + partition_session.topic_path = "/Root/orders" + message = stub_message(1) + message._partition_session = partition_session + problem = RuntimeError("decode failed") + + def fail(_): + raise problem + + stream_reader._decoders[Codec.CODEC_GZIP] = fail + batch = PublicBatch( + messages=[message], + _partition_session=partition_session, + _bytes_size=10, + _codec=Codec.CODEC_GZIP, + _received_at=20.0, + ) + + with mock.patch.object(topic_reader_asyncio, "monotonic", side_effect=[21.0, 21.25, 21.5]): + with pytest.raises(RuntimeError, match="decode failed") as raised: + await stream_reader._decode_batch_inplace(batch) + + assert raised.value is problem + assert batch._codec == Codec.CODEC_GZIP + metrics.record_compression_duration.assert_called_once_with(0.25, "gzip", "/Root/orders", problem) + metrics.record_message_processing_duration.assert_called_once_with(1.5, "gzip", "/Root/orders", problem) + + async def test_unknown_codec_records_each_message_as_failed(self, stream_reader): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + partition_session = stub_partition_session() + partition_session.topic_path = "/Root/orders" + messages = [stub_message(1), stub_message(2)] + for message in messages: + message._partition_session = partition_session + batch = PublicBatch( + messages=messages, + _partition_session=partition_session, + _bytes_size=10, + _codec=10001, + _received_at=30.0, + ) + + with mock.patch.object(topic_reader_asyncio, "monotonic", return_value=31.0): + with pytest.raises(topic_reader_asyncio.PublicTopicReaderUnexpectedCodecError) as raised: + await stream_reader._decode_batch_inplace(batch) + + error = raised.value + assert metrics.record_compression_duration.call_args_list == [ + mock.call(0.0, "10001", "/Root/orders", error), + mock.call(0.0, "10001", "/Root/orders", error), + ] + assert metrics.record_message_processing_duration.call_args_list == [ + mock.call(1.0, "10001", "/Root/orders", error), + mock.call(1.0, "10001", "/Root/orders", error), + ] + + async def test_raw_decode_does_not_record_processing_histograms(self, stream_reader): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream_reader._metrics = metrics + batch = PublicBatch( + messages=[stub_message(1)], + _partition_session=stub_partition_session(), + _bytes_size=1, + _codec=Codec.CODEC_RAW, + _received_at=10.0, + ) + + await stream_reader._decode_batch_inplace(batch) + + metrics.record_compression_duration.assert_not_called() + metrics.record_message_processing_duration.assert_not_called() + async def test_error_from_status_code(self, stream, stream_reader_finish_with_error): # noinspection PyTypeChecker stream.from_server.put_nowait( @@ -891,7 +1070,10 @@ def session_count(): reader_reconnector_id=self.default_reader_reconnector_id, reader_stream_id=stream_reader._id, ) - metrics.add_partition_session.assert_called_once_with(test_topic_path) + metrics.add_partition_session.assert_called_once_with( + test_topic_path, + stream_reader._partition_sessions[test_partition_session_id], + ) async def test_partition_stop_force(self, stream, stream_reader, partition_session): metrics = mock.Mock(spec=TopicReaderMetrics) @@ -919,7 +1101,7 @@ def session_count(): assert session_count() == initial_session_count - 1 assert partition_session.id not in stream_reader._partition_sessions - metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path) + metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path, partition_session) async def test_partition_stop_graceful(self, stream, stream_reader, partition_session): metrics = mock.Mock(spec=TopicReaderMetrics) @@ -947,7 +1129,7 @@ def session_count(): assert resp.client_message == StreamReadMessage.StopPartitionSessionResponse( partition_session_id=partition_session.id ) - metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path) + metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path, partition_session) stream.from_server.put_nowait( StreamReadMessage.FromServer( @@ -1911,9 +2093,52 @@ async def stream_create( ) reader_stream_mock_with_error.wait_error.assert_any_await() reader_stream_mock_with_error.wait_messages.assert_any_await() + metrics.record_session_error.assert_called_once_with(test_error, True) await reconnector.close(flush=False) metrics.close.assert_called_once_with() + async def test_non_retriable_connection_error_records_stop(self): + test_error = issues.BadRequest("test error") + metrics = mock.Mock(spec=TopicReaderMetrics) + reconnector = object.__new__(ReaderReconnector) + reconnector._id = 1 + reconnector._driver = mock.Mock() + reconnector._settings = PublicReaderSettings("", "", reader_name="reader") + reconnector._reader_name = "reader" + reconnector._metrics = metrics + reconnector._log_prefix = "test-reader" + reconnector._closed = False + reconnector._stream_reader = None + reconnector._first_error = asyncio.get_running_loop().create_future() + reconnector._state_changed = asyncio.Event() + + with mock.patch.object(ReaderStream, "create", mock.AsyncMock(side_effect=test_error)): + await reconnector._connection_loop() + + metrics.record_session_error.assert_called_once_with(test_error, False) + assert reconnector._first_error.result() is test_error + + async def test_connection_error_during_close_is_not_recorded(self): + metrics = mock.Mock(spec=TopicReaderMetrics) + reconnector = object.__new__(ReaderReconnector) + reconnector._id = 1 + reconnector._driver = mock.Mock() + reconnector._settings = PublicReaderSettings("", "", reader_name="reader") + reconnector._reader_name = "reader" + reconnector._metrics = metrics + reconnector._log_prefix = "test-reader" + reconnector._closed = False + reconnector._stream_reader = None + + async def close_during_create(*args, **kwargs): + reconnector._closed = True + raise TopicReaderStreamClosedError() + + with mock.patch.object(ReaderStream, "create", close_during_create): + await reconnector._connection_loop() + + metrics.record_session_error.assert_not_called() + @pytest.mark.parametrize( "close_error", [RuntimeError("stream close failed"), asyncio.CancelledError()], @@ -2025,6 +2250,116 @@ async def test_receive_batch_with_tx_does_not_record_failed_delivery(self): metrics.record_delivered_messages.assert_not_called() + @pytest.mark.parametrize("rpc_error", [None, RuntimeError("rpc failed")], ids=["success", "rpc-failure"]) + async def test_transaction_commit_records_queued_ranges(self, rpc_error): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream = mock.Mock(spec=ReaderStream) + stream._id = 513 + stream._partition_sessions = {} + + def batch(topic, partition_session_id, partition_id, start, end): + partition_session = stub_partition_session(partition_session_id) + partition_session.topic_path = topic + partition_session.partition_id = partition_id + stream._partition_sessions[partition_session_id] = partition_session + result = mock.Mock(spec=PublicBatch) + result._partition_session = partition_session + result._commit_get_offsets_range.return_value = OffsetsRange(start, end) + return result + + batches = [ + batch("/Root/orders", 1, 10, 5, 7), + batch("/Root/orders", 2, 20, 11, 14), + batch("/Root/refunds", 3, 30, 20, 24), + ] + tx = mock.Mock() + tx.tx_id = "tx-1" + + reconnector = object.__new__(ReaderReconnector) + reconnector._settings = PublicReaderSettings(consumer="analytics", topic="/Root/orders") + reconnector._stream_reader = stream + reconnector._tx_to_batches_map = {tx.tx_id: batches} + reconnector._metrics = metrics + reconnector._do_commit_batches_with_tx_call = mock.AsyncMock(side_effect=rpc_error) + + await reconnector._commit_batches_with_tx(tx) + + assert metrics.record_commit_queued.call_args_list == [ + mock.call(2, "/Root/orders"), + mock.call(3, "/Root/orders"), + mock.call(4, "/Root/refunds"), + ] + metrics.record_commit_offset_queued.assert_not_called() + reconnector._do_commit_batches_with_tx_call.assert_awaited_once() + if rpc_error is None: + assert reconnector._tx_to_batches_map[tx.tx_id] == batches + else: + assert tx.tx_id not in reconnector._tx_to_batches_map + + async def test_expired_transaction_commit_is_not_recorded(self): + metrics = mock.Mock(spec=TopicReaderMetrics) + stream = mock.Mock(spec=ReaderStream) + stream._id = 514 + stream._partition_sessions = {} + partition_session = stub_partition_session(1) + batch = mock.Mock(spec=PublicBatch) + batch._partition_session = partition_session + batch._commit_get_offsets_range.return_value = OffsetsRange(5, 7) + tx = mock.Mock() + tx.tx_id = "tx-1" + + reconnector = object.__new__(ReaderReconnector) + reconnector._settings = PublicReaderSettings(consumer="analytics", topic="/Root/orders") + reconnector._stream_reader = stream + reconnector._tx_to_batches_map = {tx.tx_id: [batch]} + reconnector._metrics = metrics + reconnector._do_commit_batches_with_tx_call = mock.AsyncMock() + + await reconnector._commit_batches_with_tx(tx) + + metrics.record_commit_queued.assert_not_called() + metrics.record_commit_offset_queued.assert_not_called() + reconnector._do_commit_batches_with_tx_call.assert_not_awaited() + + @pytest.mark.parametrize("commit_error", [None, RuntimeError("commit failed")], ids=["success", "failure"]) + async def test_transaction_commit_records_acknowledged_ranges_only_on_success(self, commit_error): + metrics = mock.Mock(spec=TopicReaderMetrics) + + def batch(topic, start, end): + result = mock.Mock(spec=PublicBatch) + result._partition_session = stub_partition_session(start) + result._partition_session.topic_path = topic + result._commit_get_offsets_range.return_value = OffsetsRange(start, end) + return result + + tx = mock.Mock() + tx.tx_id = "tx-1" + batches = [ + batch("/Root/orders", 5, 7), + batch("/Root/orders", 11, 14), + batch("/Root/refunds", 20, 24), + ] + reconnector = object.__new__(ReaderReconnector) + reconnector._tx_to_batches_map = {tx.tx_id: batches} + reconnector._metrics = metrics + reconnector._stream_reader = mock.Mock(spec=ReaderStream) + + await reconnector._handle_after_tx_commit(tx, commit_error) + + assert tx.tx_id not in reconnector._tx_to_batches_map + if commit_error is None: + assert metrics.record_commit_acknowledged.call_args_list == [ + mock.call(2, "/Root/orders"), + mock.call(3, "/Root/orders"), + mock.call(4, "/Root/refunds"), + ] + metrics.record_commit_offset_acknowledged.assert_not_called() + reconnector._stream_reader._set_first_error.assert_not_called() + else: + metrics.record_commit_acknowledged.assert_not_called() + metrics.record_commit_offset_acknowledged.assert_not_called() + reconnector._stream_reader._set_first_error.assert_called_once() + async def test_close_during_reconnect_does_not_hang(self): # The connection loop must stop on reader.close() even while it is closing the old # stream during a reconnect, instead of swallowing the cancellation and bringing up @@ -2153,7 +2488,14 @@ class TestReaderStreamBufferReleaseThreshold: def stream(self): return StreamMock() - async def _get_started_reader(self, stream, threshold, buffer_size_bytes, default_executor) -> ReaderStream: + async def _get_started_reader( + self, + stream, + threshold, + buffer_size_bytes, + default_executor, + metrics=None, + ) -> ReaderStream: settings = PublicReaderSettings( consumer="test-consumer", topic="test-topic", @@ -2161,10 +2503,14 @@ async def _get_started_reader(self, stream, threshold, buffer_size_bytes, defaul buffer_release_threshold=threshold, decoder_executor=default_executor, ) + reader_kwargs = {} + if metrics is not None: + reader_kwargs["metrics"] = metrics reader = ReaderStream( self.default_reader_reconnector_id, settings, reader_name="reader-%d" % self.default_reader_reconnector_id, + **reader_kwargs, ) init_message = object() start = asyncio.create_task(reader._start(stream, init_message)) @@ -2226,6 +2572,49 @@ async def test_threshold_zero_sends_immediately(self, stream, default_executor): await reader.close(False) + async def test_credit_balance_tracks_granted_consumed_and_released_bytes(self, stream, default_executor): + metrics = mock.Mock(spec=TopicReaderMetrics) + reader = await self._get_started_reader( + stream, + threshold=0.1, + buffer_size_bytes=1000, + default_executor=default_executor, + metrics=metrics, + ) + + metrics.reset_credit_balance_bytes.assert_called_once_with(1000) + + reader._buffer_consume_bytes(200) + reader._buffer_release_bytes(50) + metrics.add_credit_balance_bytes.assert_called_once_with(-200) + + reader._buffer_release_bytes(50) + assert metrics.add_credit_balance_bytes.call_args_list == [mock.call(-200), mock.call(100)] + request = stream.from_client.get_nowait() + assert request.client_message.bytes_size == 100 + + await reader.close(False) + assert metrics.reset_credit_balance_bytes.call_args_list == [mock.call(1000), mock.call()] + + async def test_credit_balance_is_not_replenished_when_read_request_fails(self, stream, default_executor): + metrics = mock.Mock(spec=TopicReaderMetrics) + reader = await self._get_started_reader( + stream, + threshold=0.1, + buffer_size_bytes=1000, + default_executor=default_executor, + metrics=metrics, + ) + stream.write = mock.Mock(side_effect=RuntimeError("write failed")) + + with pytest.raises(RuntimeError, match="write failed"): + reader._buffer_release_bytes(100) + + metrics.add_credit_balance_bytes.assert_not_called() + assert reader._buffer_size_bytes == 1000 + + await reader.close(False) + async def test_threshold_delays_release_while_messages_pending(self, stream, default_executor): """threshold=0.5: releases below threshold are not flushed, even when queue becomes empty.""" reader = await self._get_started_reader( diff --git a/ydb/observability/metrics.py b/ydb/observability/metrics.py index 942568a21..048502252 100644 --- a/ydb/observability/metrics.py +++ b/ydb/observability/metrics.py @@ -26,6 +26,7 @@ from collections import deque from typing import Any, Callable, Deque, Dict, Iterable, List, Optional, Protocol, Tuple, TypeVar, cast +from ydb import issues from ydb.observability._endpoint import split_endpoint CLIENT_OPERATION_DURATION = "ydb.client.operation.duration" @@ -42,11 +43,18 @@ TOPIC_READER_RECEIVED_MESSAGES = "ydb.topic.reader.received.messages" TOPIC_READER_RECEIVED_BYTES = "ydb.topic.reader.received.bytes" TOPIC_READER_DELIVERED_MESSAGES = "ydb.topic.reader.delivered.messages" +TOPIC_READER_COMMIT_QUEUED = "ydb.topic.reader.commit.queued" +TOPIC_READER_COMMIT_ACKNOWLEDGED = "ydb.topic.reader.commit.acknowledged" +TOPIC_READER_COMMIT_OFFSET_LAG_MAX = "ydb.topic.reader.commit_offset.lag.max" +TOPIC_READER_SESSION_ERRORS = "ydb.topic.reader.session.errors" +TOPIC_READER_COMPRESSION_DURATION = "ydb.topic.reader.compression.duration" +TOPIC_READER_MESSAGE_PROCESSING_DURATION = "ydb.topic.reader.message.processing.duration" +TOPIC_READER_CREDIT_BALANCE_BYTES = "ydb.topic.reader.credit_balance_bytes" TOPIC_READER_LOCAL_BUFFER_MESSAGES = "ydb.topic.reader.local_buffer.messages" TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX = "ydb.topic.reader.local_buffer.message_age.max" TOPIC_READER_PARTITION_SESSION_COUNT = "ydb.topic.reader.partition_session.count" -METRICS_SDK_BUILD_INFO = "ydb-sdk-metrics/0.7.0" +METRICS_SDK_BUILD_INFO = "ydb-sdk-metrics/0.14.0" DURATION_BUCKETS_SECONDS = ( 0.001, @@ -199,6 +207,15 @@ def _observe_topic_reader_local_buffer_messages() -> List[Tuple[float, Dict[str, return [(value, dict(attrs)) for attrs, value in values.items()] +def _observe_topic_reader_credit_balance_bytes() -> List[Tuple[float, Dict[str, Any]]]: + with _gauge_lock: + values: Dict[Tuple, int] = {} + for reader_metrics in _live_topic_reader_metrics: + key = tuple(sorted(reader_metrics._base_attributes.items())) + values[key] = values.get(key, 0) + reader_metrics._credit_balance_bytes + return [(value, dict(attrs)) for attrs, value in values.items()] + + def _observe_topic_reader_local_buffer_message_age_max() -> List[Tuple[float, Dict[str, Any]]]: now = time.monotonic() with _gauge_lock: @@ -226,15 +243,40 @@ def _observe_topic_reader_partition_session_count() -> List[Tuple[float, Dict[st return [(value, dict(attrs)) for attrs, value in values.items()] +def _observe_topic_reader_commit_offset_lag_max() -> List[Tuple[float, Dict[str, Any]]]: + with _gauge_lock: + values: Dict[Tuple, int] = {} + for reader_metrics in _live_topic_reader_metrics: + for topic, requested_offset, acknowledged_offset in reader_metrics._commit_offsets.values(): + attributes = dict(reader_metrics._base_attributes) + attributes["topic"] = topic + key = tuple(sorted(attributes.items())) + lag = max(0, requested_offset - acknowledged_offset) + values[key] = max(values.get(key, 0), lag) + return [(value, dict(attrs)) for attrs, value in values.items()] + + _OBSERVABLE_GAUGES = ( (QUERY_SESSION_COUNT, _observe_session_count), (QUERY_SESSION_MAX, _observe_session_max), (QUERY_SESSION_MIN, _observe_session_min), + (TOPIC_READER_CREDIT_BALANCE_BYTES, _observe_topic_reader_credit_balance_bytes), + (TOPIC_READER_COMMIT_OFFSET_LAG_MAX, _observe_topic_reader_commit_offset_lag_max), (TOPIC_READER_LOCAL_BUFFER_MESSAGES, _observe_topic_reader_local_buffer_messages), (TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, _observe_topic_reader_local_buffer_message_age_max), (TOPIC_READER_PARTITION_SESSION_COUNT, _observe_topic_reader_partition_session_count), ) +_TRANSPORT_STATUS_CODES = frozenset( + { + issues.StatusCode.CONNECTION_LOST, + issues.StatusCode.CONNECTION_FAILURE, + issues.StatusCode.DEADLINE_EXCEEDED, + issues.StatusCode.CLIENT_INTERNAL_ERROR, + issues.StatusCode.UNIMPLEMENTED, + } +) + def _guard_gauge_callback(provider: MetricsProvider, generation: int, callback: GaugeCallback) -> GaugeCallback: def guarded_callback(): @@ -313,7 +355,7 @@ def query_session_pool_name( def _metrics_build_info_tokens() -> List[str]: """Metrics' contribution to the ``x-ydb-sdk-build-info`` header. - Returns ``["ydb-sdk-metrics/0.7.0"]`` once a metrics backend is installed, + Returns ``["ydb-sdk-metrics/0.14.0"]`` once a metrics backend is installed, otherwise an empty list. Aggregated with other features by :func:`ydb.observability.sdk_build_info_tokens`. """ @@ -353,6 +395,15 @@ def _response_status_code(exception: BaseException) -> str: return type(exception).__qualname__ +def _error_type(exception: BaseException) -> str: + status = getattr(exception, "status", None) + if status in _TRANSPORT_STATUS_CODES or isinstance(exception, issues.ConnectionError): + return "transport_error" + if isinstance(exception, issues.Error): + return "ydb_error" + return type(exception).__qualname__ + + class MetricsOperation: """Metric lifecycle object for one user-visible YDB client operation. @@ -549,6 +600,8 @@ class TopicReaderMetrics: __slots__ = ( "_base_attributes", + "_commit_offsets", + "_credit_balance_bytes", "_local_buffer_messages", "_local_buffer_received_at", "_partition_session_counts", @@ -568,6 +621,8 @@ def __init__( self._base_attributes["reader.name"] = reader_name if consumer_name is not None: self._base_attributes["consumer"] = consumer_name + self._commit_offsets: Dict[Tuple[int, int], Tuple[str, int, int]] = {} + self._credit_balance_bytes = 0 self._local_buffer_messages = 0 self._local_buffer_received_at: Dict[Tuple[int, int], Deque[Tuple[float, int]]] = {} self._partition_session_counts: Dict[str, int] = dict.fromkeys(topics, 0) @@ -636,6 +691,116 @@ def record_delivered_messages(self, message_count: int, topic: str) -> None: attributes["topic"] = topic provider.add(TOPIC_READER_DELIVERED_MESSAGES, message_count, attributes) + def record_compression_duration( + self, + duration: float, + codec: str, + topic: str, + error: Optional[BaseException] = None, + ) -> None: + self._record_message_duration(TOPIC_READER_COMPRESSION_DURATION, duration, codec, topic, error) + + def record_message_processing_duration( + self, + duration: float, + codec: str, + topic: str, + error: Optional[BaseException] = None, + ) -> None: + self._record_message_duration(TOPIC_READER_MESSAGE_PROCESSING_DURATION, duration, codec, topic, error) + + def _record_message_duration( + self, + name: str, + duration: float, + codec: str, + topic: str, + error: Optional[BaseException], + ) -> None: + provider = _provider + if provider is _NOOP_PROVIDER: + return + + attributes = dict(self._base_attributes) + attributes.update( + { + "topic": topic, + "codec": codec, + "outcome": "error" if error is not None else "ok", + } + ) + if error is not None: + attributes["error.type"] = type(error).__qualname__ + provider.record(name, max(0.0, duration), attributes) + + def record_commit_queued(self, message_count: int, topic: str) -> None: + provider = _provider + if provider is _NOOP_PROVIDER: + return + + if message_count > 0: + attributes = dict(self._base_attributes) + attributes["topic"] = topic + provider.add(TOPIC_READER_COMMIT_QUEUED, message_count, attributes) + + def record_commit_acknowledged(self, message_count: int, topic: str) -> None: + provider = _provider + if provider is _NOOP_PROVIDER: + return + + if message_count > 0: + attributes = dict(self._base_attributes) + attributes["topic"] = topic + provider.add(TOPIC_READER_COMMIT_ACKNOWLEDGED, message_count, attributes) + + def record_commit_offset_queued(self, partition_session: Any, end_offset: int) -> None: + key = self._partition_session_key(partition_session) + with _gauge_lock: + if self._closed or key not in self._commit_offsets: + return + topic, requested_offset, acknowledged_offset = self._commit_offsets[key] + self._commit_offsets[key] = ( + topic, + max(requested_offset, end_offset), + acknowledged_offset, + ) + + def record_commit_offset_acknowledged(self, partition_session: Any, committed_offset: int) -> None: + key = self._partition_session_key(partition_session) + with _gauge_lock: + if self._closed or key not in self._commit_offsets: + return + topic, requested_offset, acknowledged_offset = self._commit_offsets[key] + self._commit_offsets[key] = ( + topic, + requested_offset, + max(acknowledged_offset, committed_offset), + ) + + def record_session_error(self, error: BaseException, is_retriable: bool) -> None: + provider = _provider + if provider is _NOOP_PROVIDER: + return + + attributes = dict(self._base_attributes) + attributes.update( + { + "retry_decision": "retry" if is_retriable else "stop", + "status_code": _response_status_code(error), + "error.type": _error_type(error), + } + ) + provider.add(TOPIC_READER_SESSION_ERRORS, 1, attributes) + + def add_credit_balance_bytes(self, bytes_size: int) -> None: + with _gauge_lock: + if not self._closed: + self._credit_balance_bytes += bytes_size + + def reset_credit_balance_bytes(self, bytes_size: int = 0) -> None: + with _gauge_lock: + self._credit_balance_bytes = bytes_size + def remove_local_buffer_messages(self, message_count: int, partition_session: Any) -> None: if message_count <= 0: return @@ -667,28 +832,37 @@ def reset_local_buffer_messages(self) -> None: self._local_buffer_messages = 0 self._local_buffer_received_at.clear() - def add_partition_session(self, topic: str) -> None: + def add_partition_session(self, topic: str, partition_session: Any = None) -> None: with _gauge_lock: if self._closed: return self._partition_session_counts[topic] = self._partition_session_counts.get(topic, 0) + 1 + if partition_session is not None: + key = self._partition_session_key(partition_session) + committed_offset = partition_session.committed_offset + self._commit_offsets[key] = (topic, committed_offset, committed_offset) - def remove_partition_session(self, topic: str) -> None: + def remove_partition_session(self, topic: str, partition_session: Any = None) -> None: with _gauge_lock: if self._closed or topic not in self._partition_session_counts: return self._partition_session_counts[topic] = max(0, self._partition_session_counts[topic] - 1) + if partition_session is not None: + self._commit_offsets.pop(self._partition_session_key(partition_session), None) def reset_partition_sessions(self) -> None: with _gauge_lock: for topic in self._partition_session_counts: self._partition_session_counts[topic] = 0 + self._commit_offsets.clear() def close(self) -> None: with _gauge_lock: if self._closed: return self._closed = True + self._commit_offsets.clear() + self._credit_balance_bytes = 0 self._local_buffer_messages = 0 self._local_buffer_received_at.clear() self._partition_session_counts.clear() @@ -713,16 +887,55 @@ def record_received_bytes(self, bytes_size: int) -> None: def record_delivered_messages(self, message_count: int, topic: str) -> None: pass + def record_compression_duration( + self, + duration: float, + codec: str, + topic: str, + error: Optional[BaseException] = None, + ) -> None: + pass + + def record_message_processing_duration( + self, + duration: float, + codec: str, + topic: str, + error: Optional[BaseException] = None, + ) -> None: + pass + + def record_commit_queued(self, message_count: int, topic: str) -> None: + pass + + def record_commit_acknowledged(self, message_count: int, topic: str) -> None: + pass + + def record_commit_offset_queued(self, partition_session: Any, end_offset: int) -> None: + pass + + def record_commit_offset_acknowledged(self, partition_session: Any, committed_offset: int) -> None: + pass + + def record_session_error(self, error: BaseException, is_retriable: bool) -> None: + pass + + def add_credit_balance_bytes(self, bytes_size: int) -> None: + pass + + def reset_credit_balance_bytes(self, bytes_size: int = 0) -> None: + pass + def remove_local_buffer_messages(self, message_count: int, partition_session: Any) -> None: pass def reset_local_buffer_messages(self) -> None: pass - def add_partition_session(self, topic: str) -> None: + def add_partition_session(self, topic: str, partition_session: Any = None) -> None: pass - def remove_partition_session(self, topic: str) -> None: + def remove_partition_session(self, topic: str, partition_session: Any = None) -> None: pass def reset_partition_sessions(self) -> None: diff --git a/ydb/opentelemetry/metrics_plugin.py b/ydb/opentelemetry/metrics_plugin.py index 7fa31d6d9..f83348b60 100644 --- a/ydb/opentelemetry/metrics_plugin.py +++ b/ydb/opentelemetry/metrics_plugin.py @@ -30,12 +30,19 @@ QUERY_SESSION_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DURATION, + TOPIC_READER_COMMIT_ACKNOWLEDGED, + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_COMMIT_QUEUED, + TOPIC_READER_COMPRESSION_DURATION, + TOPIC_READER_CREDIT_BALANCE_BYTES, TOPIC_READER_DELIVERED_MESSAGES, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, TOPIC_READER_LOCAL_BUFFER_MESSAGES, + TOPIC_READER_MESSAGE_PROCESSING_DURATION, TOPIC_READER_PARTITION_SESSION_COUNT, TOPIC_READER_RECEIVED_BYTES, TOPIC_READER_RECEIVED_MESSAGES, + TOPIC_READER_SESSION_ERRORS, ATTEMPT_BUCKETS, DURATION_BUCKETS_SECONDS, GaugeCallback, @@ -51,6 +58,14 @@ QUERY_SESSION_COUNT: ("{connection}", "Number of open YDB query sessions."), QUERY_SESSION_MAX: ("{connection}", "Maximum configured number of YDB query sessions."), QUERY_SESSION_MIN: ("{connection}", "Minimum configured number of YDB query sessions."), + TOPIC_READER_CREDIT_BALANCE_BYTES: ( + "By", + "The protocol credit granted to the server and not yet consumed by read responses.", + ), + TOPIC_READER_COMMIT_OFFSET_LAG_MAX: ( + "{message}", + "The maximum gap between requested and acknowledged commit offsets.", + ), TOPIC_READER_LOCAL_BUFFER_MESSAGES: ( "{message}", "The number of messages currently buffered by the reader.", @@ -64,7 +79,13 @@ "The number of active partition sessions owned by the reader.", ), } -_OBSERVABLE_GAUGE_INSTRUMENTS = frozenset({TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX}) +_OBSERVABLE_GAUGE_INSTRUMENTS = frozenset( + { + TOPIC_READER_CREDIT_BALANCE_BYTES, + TOPIC_READER_COMMIT_OFFSET_LAG_MAX, + TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, + } +) class OtelMetricsProvider: @@ -111,6 +132,20 @@ def __init__(self, meter: Meter) -> None: ), bucket_boundaries=ATTEMPT_BUCKETS, ), + TOPIC_READER_COMPRESSION_DURATION: _create_histogram( + meter, + TOPIC_READER_COMPRESSION_DURATION, + unit="s", + description="The time spent decompressing a message, excluding queueing.", + bucket_boundaries=DURATION_BUCKETS_SECONDS, + ), + TOPIC_READER_MESSAGE_PROCESSING_DURATION: _create_histogram( + meter, + TOPIC_READER_MESSAGE_PROCESSING_DURATION, + unit="s", + description="The time from receipt until a message is ready for delivery.", + bucket_boundaries=DURATION_BUCKETS_SECONDS, + ), } self._counters: Dict[str, Any] = { CLIENT_OPERATION_FAILED: meter.create_counter( @@ -138,6 +173,16 @@ def __init__(self, meter: Meter) -> None: unit="{message}", description="Number of messages delivered by the YDB topic reader to the application.", ), + TOPIC_READER_COMMIT_QUEUED: meter.create_counter( + TOPIC_READER_COMMIT_QUEUED, + unit="{message}", + description="The number of messages in commit ranges accepted by the SDK.", + ), + TOPIC_READER_COMMIT_ACKNOWLEDGED: meter.create_counter( + TOPIC_READER_COMMIT_ACKNOWLEDGED, + unit="{message}", + description="The number of messages acknowledged by regular or transactional commits.", + ), TOPIC_READER_RECEIVED_MESSAGES: meter.create_counter( TOPIC_READER_RECEIVED_MESSAGES, unit="{message}", @@ -148,6 +193,11 @@ def __init__(self, meter: Meter) -> None: unit="By", description="The protocol bytes_size received in read responses.", ), + TOPIC_READER_SESSION_ERRORS: meter.create_counter( + TOPIC_READER_SESSION_ERRORS, + unit="{error}", + description="Number of YDB topic reader session errors.", + ), } def record(self, name: str, value: float, attributes: Optional[Dict[str, Any]] = None) -> None: From 04edb9d1b1356d23a2cd8cefaf222608a5a0eee5 Mon Sep 17 00:00:00 2001 From: nastya Date: Fri, 30 Oct 2026 17:19:38 +0300 Subject: [PATCH 4/5] Add OpenTelemetry metrics verifier --- examples/opentelemetry/verify_metrics.py | 295 +++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 examples/opentelemetry/verify_metrics.py diff --git a/examples/opentelemetry/verify_metrics.py b/examples/opentelemetry/verify_metrics.py new file mode 100644 index 000000000..24b086288 --- /dev/null +++ b/examples/opentelemetry/verify_metrics.py @@ -0,0 +1,295 @@ +"""Verify that every YDB Python SDK metric reached Prometheus.""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, Tuple + + +@dataclass(frozen=True) +class MetricCheck: + description: str + prometheus_name: str + promql: str = "" + requires_error_probe: bool = False + + +LOOKBACK = "10m" + + +def _prometheus_label_value(value: str) -> str: + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def _selector(**labels: str) -> str: + return "{%s}" % ",".join('%s="%s"' % (name, _prometheus_label_value(value)) for name, value in labels.items()) + + +def _present(prometheus_name: str, selector: str) -> str: + return "count(count_over_time(%s%s[%s])) > 0" % (prometheus_name, selector, LOOKBACK) + + +def _positive(prometheus_name: str, selector: str) -> str: + return "max(max_over_time(%s%s[%s])) > 0" % (prometheus_name, selector, LOOKBACK) + + +def _metric_checks(service_instance_id: str) -> Tuple[MetricCheck, ...]: + run = _selector(service_instance_id=service_instance_id) + error_probe = _selector( + service_instance_id=service_instance_id, + reader_name="load-tank-topic-error-probe", + retry_decision="stop", + status_code="SCHEME_ERROR", + ) + return ( + MetricCheck( + "client operation duration", + "ydb_client_operation_duration_seconds_count", + _positive("ydb_client_operation_duration_seconds_count", run), + ), + MetricCheck( + "failed client operations", + "ydb_client_operation_failed_total", + _positive("ydb_client_operation_failed_total", run), + ), + MetricCheck("query session count", "ydb_query_session_count", _present("ydb_query_session_count", run)), + MetricCheck( + "closed query sessions", + "ydb_query_session_closed_total", + _positive("ydb_query_session_closed_total", run), + ), + MetricCheck( + "query session create time", + "ydb_query_session_create_time_seconds_count", + _positive("ydb_query_session_create_time_seconds_count", run), + ), + MetricCheck( + "pending query session requests", + "ydb_query_session_pending_requests", + _positive("ydb_query_session_pending_requests", run), + ), + MetricCheck( + "query session acquire timeouts", + "ydb_query_session_timeouts_total", + _positive("ydb_query_session_timeouts_total", run), + ), + MetricCheck("query session maximum", "ydb_query_session_max", _positive("ydb_query_session_max", run)), + MetricCheck("query session minimum", "ydb_query_session_min", _present("ydb_query_session_min", run)), + MetricCheck( + "retry attempts above one", + "ydb_client_retry_attempts_count", + "max(max_over_time(ydb_client_retry_attempts_sum%s[%s]) " + "- max_over_time(ydb_client_retry_attempts_count%s[%s])) > 0" % (run, LOOKBACK, run, LOOKBACK), + ), + MetricCheck( + "retry duration", + "ydb_client_retry_duration_seconds_count", + _positive("ydb_client_retry_duration_seconds_count", run), + ), + MetricCheck( + "topic messages received", + "ydb_topic_reader_received_messages_total", + _positive("ydb_topic_reader_received_messages_total", run), + ), + MetricCheck( + "topic bytes received", + "ydb_topic_reader_received_bytes_total", + _positive("ydb_topic_reader_received_bytes_total", run), + ), + MetricCheck( + "topic messages delivered", + "ydb_topic_reader_delivered_messages_total", + _positive("ydb_topic_reader_delivered_messages_total", run), + ), + MetricCheck( + "topic message decompression duration", + "ydb_topic_reader_compression_duration_seconds_count", + _positive("ydb_topic_reader_compression_duration_seconds_count", run), + ), + MetricCheck( + "topic message processing duration", + "ydb_topic_reader_message_processing_duration_seconds_count", + _positive("ydb_topic_reader_message_processing_duration_seconds_count", run), + ), + MetricCheck( + "topic commit ranges queued", + "ydb_topic_reader_commit_queued_total", + _positive("ydb_topic_reader_commit_queued_total", run), + ), + MetricCheck( + "topic commit ranges acknowledged", + "ydb_topic_reader_commit_acknowledged_total", + _positive("ydb_topic_reader_commit_acknowledged_total", run), + ), + MetricCheck( + "topic commit offset lag", + "ydb_topic_reader_commit_offset_lag_max", + _present("ydb_topic_reader_commit_offset_lag_max", run), + ), + MetricCheck( + "topic reader session errors", + "ydb_topic_reader_session_errors_total", + _positive("ydb_topic_reader_session_errors_total", error_probe), + requires_error_probe=True, + ), + MetricCheck( + "topic reader credit balance", + "ydb_topic_reader_credit_balance_bytes", + _positive("ydb_topic_reader_credit_balance_bytes", run), + ), + MetricCheck( + "topic reader local buffer", + "ydb_topic_reader_local_buffer_messages", + _positive("ydb_topic_reader_local_buffer_messages", run), + ), + MetricCheck( + "oldest buffered topic message age", + "ydb_topic_reader_local_buffer_message_age_max_seconds", + _positive("ydb_topic_reader_local_buffer_message_age_max_seconds", run), + ), + MetricCheck( + "topic partition sessions", + "ydb_topic_reader_partition_session_count", + _positive("ydb_topic_reader_partition_session_count", run), + ), + ) + + +def _prometheus_query_has_result(prometheus_url: str, promql: str, timeout: float = 5) -> bool: + query = urllib.parse.urlencode({"query": promql}) + url = "%s/api/v1/query?%s" % (prometheus_url.rstrip("/"), query) + with urllib.request.urlopen(url, timeout=timeout) as response: + payload = json.load(response) + return payload.get("status") == "success" and bool(payload.get("data", {}).get("result")) + + +def _prometheus_has_series(prometheus_url: str, prometheus_name: str) -> bool: + return _prometheus_query_has_result(prometheus_url, "count(%s)" % prometheus_name) + + +def wait_for_metrics( + prometheus_url: str, + checks: Iterable[MetricCheck], + timeout: float, + poll_interval: float, +) -> Dict[str, MetricCheck]: + missing = {check.prometheus_name: check for check in checks} + deadline = time.monotonic() + timeout + + while missing and time.monotonic() < deadline: + for prometheus_name in tuple(missing): + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + check = missing[prometheus_name] + if _prometheus_query_has_result( + prometheus_url, + check.promql or "count(%s)" % prometheus_name, + timeout=min(5, remaining), + ): + check = missing.pop(prometheus_name) + print("[ok] %s: %s" % (check.description, check.prometheus_name)) + except (OSError, ValueError, json.JSONDecodeError): + pass + + if missing: + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(poll_interval, remaining)) + + return missing + + +def _read_run_id_file(path: str, timeout: float, poll_interval: float) -> str: + deadline = time.monotonic() + timeout + while True: + try: + run_id = Path(path).read_text(encoding="utf-8").strip() + if run_id: + return run_id + except OSError: + pass + + remaining = deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError("Run id was not published to %s" % path) + time.sleep(min(poll_interval, remaining)) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--prometheus-url", + default=os.environ.get("PROMETHEUS_URL", "http://localhost:9090"), + ) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--poll-interval", type=float, default=2) + parser.add_argument( + "--service-instance-id", + default=os.environ.get("METRICS_SERVICE_INSTANCE_ID", ""), + help="service.instance.id of the load generator run to verify", + ) + parser.add_argument( + "--run-id-file", + default=os.environ.get("METRICS_RUN_ID_FILE", ""), + help="file from which to read the current load generator run id", + ) + parser.add_argument( + "--require-error-probe", + action="store_true", + default=os.environ.get("REQUIRE_TOPIC_ERROR_PROBE", "").lower() in {"1", "true", "yes"}, + help="also require ydb.topic.reader.session.errors", + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + started_at = time.monotonic() + if not args.service_instance_id and not args.run_id_file: + print("Specify --service-instance-id or --run-id-file to select the load generator run.") + return 2 + + try: + service_instance_id = args.service_instance_id or _read_run_id_file( + args.run_id_file, + args.timeout, + args.poll_interval, + ) + except RuntimeError as error: + print("Cannot select the load generator run: %s" % error) + return 2 + + checks = tuple( + check + for check in _metric_checks(service_instance_id) + if args.require_error_probe or not check.requires_error_probe + ) + remaining_timeout = max(0, args.timeout - (time.monotonic() - started_at)) + print("Checking metrics for service.instance.id=%s" % service_instance_id) + missing = wait_for_metrics( + args.prometheus_url, + checks, + timeout=remaining_timeout, + poll_interval=args.poll_interval, + ) + if not missing: + print("All %s expected SDK metrics are present in Prometheus." % len(checks)) + return 0 + + print("Missing SDK metrics after %ss:" % args.timeout) + for check in missing.values(): + print("[missing] %s: %s" % (check.description, check.prometheus_name)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From d0a389484f3d0a2c43605cffc241e1989da47db7 Mon Sep 17 00:00:00 2001 From: nastya Date: Fri, 30 Oct 2026 17:28:20 +0300 Subject: [PATCH 5/5] Fix metrics test formatting --- tests/observability/test_metrics.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/observability/test_metrics.py b/tests/observability/test_metrics.py index aaa7be862..407f26a6d 100644 --- a/tests/observability/test_metrics.py +++ b/tests/observability/test_metrics.py @@ -3158,10 +3158,7 @@ def test_opentelemetry_dashboard_keeps_topic_reader_instances_separate(): assert topic_reader_targets for target in topic_reader_targets: - assert all( - label in target["expr"] - for label in ("service_instance_id", "endpoint", "database", "consumer") - ) + assert all(label in target["expr"] for label in ("service_instance_id", "endpoint", "database", "consumer")) def test_opentelemetry_load_tank_publishes_current_run_id():