diff --git a/.dockerignore b/.dockerignore
index 418b31eb..e48815d6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -7,4 +7,5 @@
!setup.py
!examples/opentelemetry/requirements.txt
!examples/opentelemetry/otel_example.py
-!examples/opentelemetry/load_tank.py
\ No newline at end of file
+!examples/opentelemetry/load_tank.py
+!examples/opentelemetry/verify_metrics.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7a8302ae..c3643ae9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,6 @@
+* Add `ydb.topic.writer.sending.messages`, `ydb.topic.writer.sending.bytes`, `ydb.topic.writer.written.messages`, and `ydb.topic.writer.session.errors` counters, `ydb.topic.writer.buffer.used.bytes`, `ydb.topic.writer.buffer.limit.bytes`, and `ydb.topic.writer.sending.oldest_age` observable gauges, and `ydb.topic.writer.buffer.wait.duration`, `ydb.topic.writer.message.processing.duration`, `ydb.topic.writer.compression.duration`, and `ydb.topic.writer.message.ack.duration` histograms for explicitly named topic writers; metrics-enabled clients now advertise `ydb-sdk-metrics/0.25.0` in `x-ydb-sdk-build-info`
+* Add `ydb.topic.reader.received.messages`, reader-level `ydb.topic.reader.received.bytes`, `ydb.topic.reader.delivered.messages`, `ydb.topic.reader.commit.queued`, `ydb.topic.reader.commit.acknowledged`, and `ydb.topic.reader.session.errors` counters, `ydb.topic.reader.compression.duration` and `ydb.topic.reader.message.processing.duration` histograms, plus `ydb.topic.reader.commit_offset.lag.max`, `ydb.topic.reader.credit_balance_bytes`, `ydb.topic.reader.local_buffer.messages`, `ydb.topic.reader.local_buffer.message_age.max`, and `ydb.topic.reader.partition_session.count` observable metrics for explicitly named topic readers; metrics-enabled clients now advertise `ydb-sdk-metrics/0.14.0` in `x-ydb-sdk-build-info`
+* Add optional topic reader names for identifying logical readers in server diagnostics and SDK logs
## 3.35.0 ##
* Include streaming queries in scheme directory listings and path descriptions
* Add timeout to the token exchange request so a stalled token endpoint cannot hang credential refresh
diff --git a/docs/observability.rst b/docs/observability.rst
index f332cf7f..f58d45f2 100644
--- a/docs/observability.rst
+++ b/docs/observability.rst
@@ -268,17 +268,18 @@ Client-side metrics are enabled independently of tracing:
disable_metrics() # turn metrics off — back to the no-op default
Operation and retry metrics follow the currently active provider immediately. Call
-``enable_metrics`` before creating query sessions or a ``QuerySessionPool`` whose
-lifecycle should be instrumented. Session and pool lifecycle trackers created while
-metrics are disabled keep shared no-op instrumentation and are not retrofitted later.
+``enable_metrics`` before creating query sessions, a ``QuerySessionPool``, or a topic
+Reader or Writer whose lifecycle should be instrumented. Session, pool, Reader, and
+Writer lifecycle trackers created while metrics are disabled keep shared no-op
+instrumentation and are not retrofitted later.
Calling ``enable_metrics`` again replaces the active backend. New events go to the new
backend, while operations and wait timers already in flight finish on the backend on
which they started. Observable callbacks registered by an old backend become inactive.
-Pools that were created while metrics were active retain their lightweight lifecycle
-state across ``disable_metrics`` / re-enable, so a replacement backend observes their
-current values. Pools created while metrics were disabled retain no-op lifecycle
-instrumentation.
+Pools, topic Readers, and topic Writers that were created while metrics were active
+retain their lightweight lifecycle state across ``disable_metrics`` / re-enable, so a
+replacement backend observes their current values. Pools, Readers, and Writers created
+while metrics were disabled retain no-op lifecycle instrumentation.
For OpenTelemetry the built-in convenience is ``ydb.opentelemetry.enable_metrics``,
which builds an OTel-backed provider from a meter provider and installs it here — see
@@ -352,6 +353,78 @@ adapter maps them to instruments on the ``"ydb.sdk"`` meter):
* - ``ydb.client.retry.attempts``
- Histogram
- Number of attempts performed for one logical retried operation.
+ * - ``ydb.topic.writer.sending.messages``
+ - Counter (``{message}``)
+ - Messages accepted by the SDK from the application for sending.
+ * - ``ydb.topic.writer.sending.bytes``
+ - Counter (``By``)
+ - Uncompressed message body bytes accepted by the SDK, excluding metadata and envelope overhead.
+ * - ``ydb.topic.writer.written.messages``
+ - Counter (``{message}``)
+ - Messages confirmed by the server, including already-written and transactional messages.
+ * - ``ydb.topic.writer.session.errors``
+ - Counter (``{error}``)
+ - Writer session errors by retry decision, status code, and error type.
+ * - ``ydb.topic.writer.buffer.used.bytes``
+ - ObservableGauge (``By``)
+ - Current occupied budget of the Writer buffer limiter.
+ * - ``ydb.topic.writer.buffer.limit.bytes``
+ - ObservableGauge (``By``)
+ - Configured byte limit of the Writer buffer limiter.
+ * - ``ydb.topic.writer.buffer.wait.duration``
+ - Histogram (``s``)
+ - Time spent waiting for buffer capacity before a Write batch is accepted.
+ * - ``ydb.topic.writer.message.processing.duration``
+ - Histogram (``s``)
+ - Time from buffer acceptance until a message is ready for sending.
+ * - ``ydb.topic.writer.compression.duration``
+ - Histogram (``s``)
+ - Time spent in one Writer compressor invocation, excluding queueing.
+ * - ``ydb.topic.writer.message.ack.duration``
+ - Histogram (``s``)
+ - Time from the first transport send until the server acknowledges a message.
+ * - ``ydb.topic.writer.sending.oldest_age``
+ - ObservableGauge (``s``)
+ - Age of the oldest accepted message that has not completed.
+ * - ``ydb.topic.reader.received.messages``
+ - Counter (``{message}``)
+ - Messages received by the SDK topic Reader before decoding.
+ * - ``ydb.topic.reader.received.bytes``
+ - Counter (``By``)
+ - Total ``ReadResponse`` bytes received by the SDK topic Reader.
+ * - ``ydb.topic.reader.delivered.messages``
+ - Counter (``{message}``)
+ - Messages returned by the topic Reader to the application.
+ * - ``ydb.topic.reader.compression.duration``
+ - Histogram (``s``)
+ - Time spent decompressing a message, excluding decoder-executor queueing.
+ * - ``ydb.topic.reader.message.processing.duration``
+ - Histogram (``s``)
+ - Time from receipt until a message is ready for delivery.
+ * - ``ydb.topic.reader.commit.queued``
+ - Counter (``{message}``)
+ - Messages in commit ranges accepted by the SDK.
+ * - ``ydb.topic.reader.commit.acknowledged``
+ - Counter (``{message}``)
+ - Messages acknowledged by regular committed-offset advances or successful transactions.
+ * - ``ydb.topic.reader.commit_offset.lag.max``
+ - ObservableGauge (``{message}``)
+ - Maximum gap between requested and acknowledged commit offsets across active partitions.
+ * - ``ydb.topic.reader.session.errors``
+ - Counter (``{error}``)
+ - Reader session errors by retry decision, status code, and error type.
+ * - ``ydb.topic.reader.credit_balance_bytes``
+ - ObservableGauge (``By``)
+ - Protocol credit granted to the server and not yet consumed by read responses.
+ * - ``ydb.topic.reader.local_buffer.messages``
+ - ObservableUpDownCounter (``{message}``)
+ - Current number of messages owned by the topic Reader and not yet returned to the application.
+ * - ``ydb.topic.reader.local_buffer.message_age.max``
+ - ObservableGauge (``s``)
+ - Age of the oldest message owned by the topic Reader and not yet returned to the application.
+ * - ``ydb.topic.reader.partition_session.count``
+ - ObservableUpDownCounter (``{session}``)
+ - Current number of active partition sessions owned by the topic Reader.
Attributes
~~~~~~~~~~
@@ -402,6 +475,189 @@ initial attach handshake does not count as closing an active session, and standa
``QuerySession`` instances do not publish pool metrics. A session publishes at most one
closure event; the first terminal reason wins.
+Topic Writer Metrics
+~~~~~~~~~~~~~~~~~~~~
+
+Writer metrics carry ``endpoint``, ``database``, ``topic``, and ``writer.name``. They are published only
+when the application passes a non-empty ``writer_name`` to ``TopicClient.writer`` or
+``TopicClient.tx_writer``. ``ydb.topic.writer.sending.messages`` and
+``ydb.topic.writer.sending.bytes`` are updated after a Write call passes
+validation and any buffer wait, once the messages are accepted into the SDK's
+processing buffer. The byte counter uses the uncompressed body size and excludes
+metadata and envelope overhead. Failures before acceptance do not count; retries and
+stream reconnects do not increment either counter again.
+
+``ydb.topic.writer.written.messages`` is updated once for each message acknowledged
+by the server. Written, already-written/skipped, and transactional acknowledgements
+all count; a transactional acknowledgement does not imply that the transaction was
+committed. The counter has no ``status`` attribute. Retries and reconnects do not
+count an acknowledgement twice.
+
+``ydb.topic.writer.message.ack.duration`` records one measurement for each server
+acknowledgement. Its monotonic timer starts immediately before the message's first
+transport send. Reconnects and resends keep the original start time; a message that
+ends with an error or Writer shutdown without an acknowledgement produces no sample.
+The histogram carries only the common Writer attributes.
+
+``ydb.topic.writer.sending.oldest_age`` reports the monotonic age of the oldest
+accepted message that has not yet completed. The lifetime starts after validation and
+buffer admission, includes processing, stream waits, retries, and reconnect backoff,
+and ends on server acknowledgement or terminal Writer failure. A live Writer with no
+unfinished messages reports ``0``. Writers with identical attributes are aggregated
+by maximum age, and closing a Writer removes its contribution.
+
+Each failed session or connection attempt increments ``ydb.topic.writer.session.errors``
+once, regardless of how many messages are affected. The ``retry_decision`` attribute is
+``retry`` only when the Writer will reconnect and ``stop`` when it terminates; a
+transactional Writer therefore reports ``stop`` even for an otherwise retriable error.
+``status_code`` contains the YDB status name when available and otherwise the exception
+class name. ``error.type`` is ``transport_error`` for transport failures, ``ydb_error``
+for other YDB errors, and the exception class name for other failures. Validation and
+encoding failures are not session errors, and closing a Writer normally does not
+increment the counter.
+
+``ydb.topic.writer.buffer.used.bytes`` reports the current buffer-limiter budget
+reserved by accepted messages. It uses the same accounting as Writer backpressure:
+uncompressed body bytes, metadata, and per-message envelope overhead. The value is
+reserved only after any buffer wait succeeds and is released after the corresponding
+server acknowledgement. A soft limit can therefore be exceeded temporarily when a
+whole batch or one oversized message is admitted. Reconnecting does not reserve the
+same messages again or reset the value. Writers with identical attributes are summed,
+and closing a Writer removes its contribution. When neither byte nor message
+backpressure is configured, there is no active limiter and the value remains ``0``.
+
+``ydb.topic.writer.buffer.limit.bytes`` reports ``max_buffer_size_bytes`` from the
+same limiter. The value is stable for the Writer lifetime and may be lower than
+``buffer.used.bytes`` temporarily because the Python Writer uses soft-limit admission.
+When ``max_buffer_size_bytes`` is ``None`` (including a Writer limited only by message
+count), the byte-limit series is absent rather than reporting a misleading ``0``.
+Finite limits from Writers with identical attributes are summed; if any such Writer is
+unbounded, the aggregate limit series is absent. Closing a Writer removes its current
+limit contribution.
+
+``ydb.topic.writer.buffer.wait.duration`` records one measurement when a Write batch
+that was blocked by byte or message backpressure is successfully admitted. The
+monotonic timer starts on the first full-buffer observation and spans repeated wakeups
+until the same batch is accepted. Immediate admission produces no sample, and a wait
+ending in timeout, cancellation, or Writer shutdown remains incomplete and produces no
+sample. Python admits a Write batch atomically, so one admitted batch produces one
+measurement regardless of its message count. The wait ends before the ``sending``
+counters and message-processing lifetime begin.
+
+``ydb.topic.writer.message.processing.duration`` records one measurement per accepted
+message, from successful buffer admission until its body is ready for transport. It
+includes the Python encoding-queue delay, codec selection, and compression, but excludes
+the preceding buffer wait, later stream-readiness waits, transport, retries, and server
+acknowledgement. RAW messages complete on the direct ready path. ``codec`` identifies
+the selected codec, ``outcome`` is ``ok`` or ``error``, and failures also carry
+``error.type``. A processing failure completes the measurement for every message the
+Python batch fails; cancellation or shutdown before readiness leaves it incomplete and
+produces no sample. Reconnect and resend do not repeat processing measurements for
+already prepared messages.
+
+``ydb.topic.writer.compression.duration`` records one measurement for every non-RAW
+encoder invocation. The monotonic timer runs inside the encoder worker and therefore
+excludes time waiting in the encoding executor queue. Python's adaptive codec selection
+may compress a message speculatively before the final encoding; each such invocation is
+reported separately, as is a later repeated compression. ``codec`` identifies the
+invoked encoder, ``outcome`` is ``ok`` or ``error``, and failures also carry
+``error.type``. RAW handling, reuse or resend of already encoded bytes, and cancellation
+before an encoder starts produce no measurement.
+
+.. _topic-reader-receive-metrics:
+
+Topic Reader Metrics
+~~~~~~~~~~~~~~~~~~~~
+
+Topic Reader metrics carry ``endpoint``, ``database``, and
+``reader.name``. The ``consumer`` attribute is present only for consumer-based reads;
+the SDK omits it when reading without a consumer. ``ydb.topic.reader.received.messages``,
+``ydb.topic.reader.delivered.messages``, ``ydb.topic.reader.compression.duration``,
+``ydb.topic.reader.message.processing.duration``, ``ydb.topic.reader.commit.queued``,
+``ydb.topic.reader.commit.acknowledged``, ``ydb.topic.reader.commit_offset.lag.max``,
+and ``ydb.topic.reader.partition_session.count`` additionally carry ``topic`` because
+each message, commit offset, and partition session has an unambiguous topic. The reader-level
+``ydb.topic.reader.received.bytes``, ``ydb.topic.reader.session.errors``,
+``ydb.topic.reader.credit_balance_bytes``,
+``ydb.topic.reader.local_buffer.messages``, and
+``ydb.topic.reader.local_buffer.message_age.max`` metrics omit ``topic``: one
+``ReadResponse`` can contain data from several topics, and the buffer gauges follow the
+total retained by the logical Reader, while a session error belongs to the whole Reader.
+Each response increments the byte counter once by ``ReadResponse.bytes_size``.
+Metrics are published only when the application passes a non-empty ``reader_name`` to
+``TopicClient.reader``. The name and metric context belong to the logical Reader and
+remain stable across stream reconnects. The byte counter is updated as soon as the
+transport delivers a ``ReadResponse`` to the SDK, before the response is converted to
+batches; it is therefore counted even if conversion later fails. The message counter is
+updated after the batches are created and queued, but before their payloads are decoded.
+For a multi-topic Reader, message increments are
+separated by ``topic`` while the response byte increment remains a single Reader-level
+value. The delivery counter is updated only after a message or batch has been removed
+from the SDK buffer and is ready to be returned by ``receive_message``, ``receive_batch``,
+or ``receive_batch_with_tx``. A batch increments it by the number of messages actually
+returned after applying ``max_messages`` and ``max_bytes``. Empty reads, timeouts, and
+failures before the return point do not increment it. The local-buffer gauge increases
+when accepted batches enter the decoding queue and decreases when messages leave the
+internal buffer, including partial batch reads. It therefore includes both messages
+waiting for decoding and decoded messages waiting for the application.
+For every non-RAW message, ``ydb.topic.reader.compression.duration`` records only time
+inside the decoder callable; time waiting in the decoder executor is excluded.
+``ydb.topic.reader.message.processing.duration`` starts when the ``ReadResponse`` is
+received and ends when the decoded batch becomes ready for delivery, so it includes
+response conversion and decoder queueing. Both histograms carry ``topic``, ``codec``,
+and ``outcome=ok|error``; failed measurements also carry ``error.type``. A batch is the
+delivery unit in the Python SDK, so every message in one batch receives the same
+processing duration. If any decoder fails, the batch's processing measurements use
+``outcome=error``. RAW messages require no decoding and do not create either histogram
+measurement. Only completed measurements are recorded.
+Each commit range accepted by the SDK increments ``ydb.topic.reader.commit.queued`` by
+``range.end - range.start``. A regular commit is counted after its
+``CommitOffsetRequest`` is queued on the active stream. A transactional commit is counted
+after its ranges pass SDK validation and are submitted to the transaction update path.
+Already acknowledged ranges, expired partition sessions, and regular commit requests
+that cannot be queued are not counted. A later transport or transaction failure does not
+undo this cumulative counter.
+For a regular ``CommitOffsetResponse``, ``ydb.topic.reader.commit.acknowledged`` increments
+by ``max(0, committed_offset - previous_committed_offset)``. This counts the monotonic
+committed-offset advance once even when queued ranges overlap. Repeated and stale
+responses add nothing. Transactional ranges are acknowledged by their sizes only after
+the whole transaction commits successfully; a failed commit or rollback does not
+increment the counter.
+``ydb.topic.reader.commit_offset.lag.max`` reports the maximum, across active partition
+sessions of each topic, of ``max(0, highest_requested_end_offset - acknowledged_offset)``.
+It increases after a regular ``CommitOffsetRequest`` is queued and decreases after a
+``CommitOffsetResponse`` advances the acknowledged offset. Transactional commit ranges
+remain visible in the commit counters but do not contribute to this stream-ACK gauge.
+A stale acknowledgement cannot increase it. Stopping a partition session removes that
+partition's contribution; reconnecting replaces the old stream's state, and a topic
+with no active partition sessions has no observation.
+Each failed session or connection attempt increments ``ydb.topic.reader.session.errors``
+once after the retry policy classifies it. The ``retry_decision`` attribute is ``retry``
+when the Reader will reconnect and ``stop`` when the error terminates the Reader.
+``status_code`` contains the YDB status name when available and otherwise the exception
+class name. ``error.type`` is ``transport_error`` for transport failures, ``ydb_error``
+for other YDB errors, and the exception class name for other failures. Closing a Reader
+normally does not increment the counter.
+``credit_balance_bytes`` is set to the configured buffer size after the initial
+``ReadRequest`` is sent, decreases by ``ReadResponse.bytes_size``, and increases only
+when the Reader sends another ``ReadRequest`` after its release threshold is reached.
+It is ``0`` while the logical Reader has no active stream, including reconnect gaps.
+``local_buffer.message_age.max`` is computed with a monotonic clock from the receive
+time of the oldest retained message; it is ``0`` when the buffer is empty. Receipt
+cohorts are tracked internally per partition session so delivery from another partition
+does not advance the oldest timestamp incorrectly. ``partition_session.count`` changes
+when the server starts or stops a partition session, is split by ``topic``, and does not
+carry a ``partition`` attribute. Each configured topic is observable with value ``0``
+even before its first partition session starts. Remaining messages and partition
+sessions are reset when a stream closes or reconnects, and closing the logical Reader
+removes its observations. Use rates of the cumulative counters to compare incoming and
+application-visible traffic; read the observable instruments as current values. For
+matching Reader attributes, credit balances, buffer counts, and partition-session counts
+are summed while the maximum buffer age is published.
+
+See :ref:`opentelemetry-topic-reader-metrics` for an OpenTelemetry setup example. A
+complete runnable version is available in ``examples/opentelemetry/otel_example.py``.
+
Writing a Custom Metrics Backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -443,12 +699,17 @@ subclass; just match the signatures:
The instrument kind for each metric is implied by which method the SDK calls: names in
the *Histogram* rows above arrive through ``record``, *Counter* / *UpDownCounter* names
-through ``add``, and the three *ObservableUpDownCounter* gauges
-(``ydb.query.session.count`` / ``max`` / ``min``) through ``observe_gauge``. A backend
+through ``add``, and the eleven observable instruments (the three query-session gauges,
+``ydb.topic.reader.commit_offset.lag.max``,
+``ydb.topic.reader.credit_balance_bytes``, ``ydb.topic.reader.local_buffer.messages``,
+``ydb.topic.reader.local_buffer.message_age.max``, and
+``ydb.topic.reader.partition_session.count``, plus
+``ydb.topic.writer.buffer.used.bytes``, ``ydb.topic.writer.buffer.limit.bytes``, and
+``ydb.topic.writer.sending.oldest_age``) through ``observe_gauge``. A backend
that has no notion of asynchronous gauges can simply store the callbacks (or ignore
them); the SDK never pushes those values, so nothing is lost elsewhere.
``disable_metrics()`` reverts event recording to the no-op default and deactivates the
-current backend's gauge callbacks. Trackers belonging to already instrumented pools
-remain available so a replacement backend starts with an accurate snapshot; pool and
-session lifecycle trackers that started on the no-op path remain no-ops.
+current backend's gauge callbacks. Trackers belonging to already instrumented pools,
+topic Readers, and topic Writers remain available so a replacement backend starts with
+an accurate snapshot; lifecycle trackers that started on the no-op path remain no-ops.
diff --git a/docs/opentelemetry.rst b/docs/opentelemetry.rst
index 0dc8d6dc..cb605f5f 100644
--- a/docs/opentelemetry.rst
+++ b/docs/opentelemetry.rst
@@ -8,12 +8,14 @@ driver initialization, and retries — into OpenTelemetry spans, and propagates
context to the YDB server through gRPC metadata using the
`W3C Trace Context `_ standard. Enabling metrics
records client-side OpenTelemetry instruments for the same operations, retries, and
-query session pools.
+query session pools, as well as receive, delivery, and buffer metrics for explicitly
+named topic Readers and Writers.
Like every backend, it has a **zero-cost no-op path when disabled**: operations, query
-sessions, and pools share no-op instrumentation without metric allocations, timers,
-locks, or state tracking. The SDK does not import ``opentelemetry`` until you call
-``enable_tracing()`` or ``enable_metrics()``.
+sessions, pools, and topic Readers share no-op instrumentation without metric
+allocations, timers, locks, or state tracking; topic Writers use the same no-op path.
+The SDK does not import
+``opentelemetry`` until you call ``enable_tracing()`` or ``enable_metrics()``.
Installation
@@ -139,6 +141,183 @@ The full list of instruments and their attributes is catalogued on the
:doc:`observability` page.
+Topic Writer Metrics
+--------------------
+
+Enable metrics before creating the Writer and pass a stable, non-empty
+``writer_name``:
+
+.. code-block:: python
+
+ with driver.topic_client.writer(
+ "/local/orders",
+ writer_name="orders-writer",
+ ) as writer:
+ writer.write(["one", "two"])
+
+``ydb.topic.writer.sending.messages`` counts messages accepted from the application
+after validation and any buffer wait. It carries ``endpoint``, ``database``, ``topic``,
+and ``writer.name``. ``ydb.topic.writer.sending.bytes`` counts their uncompressed body
+bytes, excluding metadata and envelope overhead, and carries the same attributes.
+``ydb.topic.writer.written.messages`` counts server acknowledgements for written,
+already-written/skipped, and transactional messages. It carries the same attributes
+without a ``status`` attribute; a transactional acknowledgement does not imply commit.
+``ydb.topic.writer.session.errors`` counts session and connection failures once each.
+It adds ``retry_decision=retry|stop``, ``status_code``, and ``error.type``; transactional
+Writers report ``stop`` because they do not reconnect. Normal shutdown and validation
+or encoding failures are not counted. Retries and reconnects do not count a message or
+acknowledgement again. ``ydb.topic.writer.buffer.used.bytes`` is an observable gauge
+of the current buffer-limiter reservation. It follows the Writer's backpressure
+accounting, including metadata and envelope overhead, is released on acknowledgement,
+and is not reset by reconnects. ``ydb.topic.writer.buffer.limit.bytes`` reports the
+configured ``max_buffer_size_bytes`` from the same limiter. It is absent when that
+limit is ``None`` instead of reporting ``0`` for an unbounded Writer. Closing the
+Writer removes both gauge series. ``ydb.topic.writer.buffer.wait.duration`` is a
+histogram in seconds. It records one monotonic-clock measurement only when a Write
+batch that actually waited for byte or message capacity is accepted; immediate
+admission and waits ending in timeout, cancellation, or shutdown produce no sample.
+Repeated wakeups remain one measurement.
+``ydb.topic.writer.message.processing.duration`` records one histogram measurement per
+accepted message from buffer admission until transport-ready data is available. It
+includes queueing, codec selection, and compression, and uses ``codec`` plus
+``outcome=ok|error`` attributes; failures also include ``error.type``. Buffer wait,
+stream readiness, retries, and acknowledgement are excluded.
+``ydb.topic.writer.compression.duration`` records each non-RAW encoder invocation and
+excludes executor queueing. Adaptive codec-selection trials and repeated compression
+produce separate measurements with the same ``codec``, ``outcome``, and optional
+``error.type`` attributes. Resending already encoded data produces no new measurement.
+``ydb.topic.writer.message.ack.duration`` records one monotonic-clock measurement per
+server acknowledgement, from the first transport send; retries and reconnects do not
+reset the start time, and messages ending without an acknowledgement produce no sample.
+``ydb.topic.writer.sending.oldest_age`` is an observable gauge for the oldest accepted,
+unfinished message. It includes processing and reconnect waits, reports ``0`` when the
+Writer has no unfinished messages, and aggregates identical Writer attributes by
+maximum age. If ``writer_name`` is omitted or empty, the Writer works normally but does
+not publish these metrics.
+
+
+.. _opentelemetry-topic-reader-metrics:
+
+Topic Reader Metrics
+--------------------
+
+Enable metrics before creating the Reader and pass an explicit, non-empty
+``reader_name``. The name identifies one logical Reader and remains the same when its
+underlying stream reconnects:
+
+.. code-block:: python
+
+ import asyncio
+ import uuid
+
+ from opentelemetry.sdk.metrics import MeterProvider
+ from opentelemetry.sdk.metrics.export import (
+ ConsoleMetricExporter,
+ PeriodicExportingMetricReader,
+ )
+
+ import ydb
+ from ydb.opentelemetry import enable_metrics
+
+ metric_reader = PeriodicExportingMetricReader(
+ ConsoleMetricExporter(),
+ export_interval_millis=1000,
+ )
+ meter_provider = MeterProvider(metric_readers=[metric_reader])
+ enable_metrics(meter_provider)
+
+
+ async def read_one_message():
+ topic_path = f"/local/otel-reader-metrics-example-{uuid.uuid4().hex}"
+ consumer = "orders-consumer"
+
+ async with ydb.aio.Driver(
+ endpoint="grpc://localhost:2136",
+ database="/local",
+ ) as driver:
+ await driver.wait(timeout=5)
+
+ await driver.topic_client.create_topic(topic_path, consumers=[consumer])
+ try:
+ async with driver.topic_client.writer(
+ topic_path,
+ codec=ydb.TopicCodec.GZIP,
+ writer_name="payments-writer",
+ ) as writer:
+ await writer.write("OpenTelemetry topic Reader metrics")
+
+ async with driver.topic_client.reader(
+ topic_path,
+ consumer=consumer,
+ reader_name="payments-worker",
+ ) as reader:
+ await asyncio.wait_for(reader.wait_message(), timeout=10)
+ await asyncio.sleep(2)
+ message = await asyncio.wait_for(reader.receive_message(), timeout=10)
+ await reader.commit_with_ack(message)
+ await asyncio.sleep(2)
+ finally:
+ await driver.topic_client.drop_topic(topic_path)
+
+
+ try:
+ asyncio.run(read_one_message())
+ finally:
+ meter_provider.shutdown()
+
+This registers six cumulative counters, two histograms, and five observable instruments:
+
+* ``ydb.topic.reader.received.messages`` counts received messages before decoding. It
+ includes ``endpoint``, ``database``, ``reader.name``, optional ``consumer``, and
+ ``topic`` attributes.
+* ``ydb.topic.reader.received.bytes`` adds ``ReadResponse.bytes_size`` once for every
+ response received by the logical Reader. It has the same Reader attributes but no
+ ``topic``, because one response may contain batches from several topics.
+* ``ydb.topic.reader.delivered.messages`` counts messages returned to the application
+ by ``receive_message``, ``receive_batch``, or ``receive_batch_with_tx``. It carries
+ the Reader attributes and ``topic``.
+* ``ydb.topic.reader.compression.duration`` records time spent inside the decoder for
+ each non-RAW message, excluding decoder-executor queueing. It uses seconds and carries
+ ``topic``, ``codec``, ``outcome``, and ``error.type`` on failures.
+* ``ydb.topic.reader.message.processing.duration`` records time from receipt until each
+ non-RAW message's batch is ready for delivery. It includes decoder queueing and uses
+ the same attributes as the compression histogram.
+* ``ydb.topic.reader.commit.queued`` counts messages in commit ranges accepted by the
+ SDK, using ``range.end - range.start``. It carries the Reader attributes and ``topic``.
+* ``ydb.topic.reader.commit.acknowledged`` counts monotonic committed-offset advances
+ reported by regular acknowledgements. Successful transaction commits count their
+ submitted ranges; duplicate or stale regular acknowledgements and failed transactions
+ add nothing. It carries the Reader attributes and ``topic``.
+* ``ydb.topic.reader.commit_offset.lag.max`` reports the maximum gap between the highest
+ requested commit end offset and the acknowledged offset across active partitions of
+ each ``topic``. It tracks regular stream commits; transactional ranges remain visible
+ in the commit counters but do not contribute to this gauge. Stale acknowledgements
+ cannot increase it.
+* ``ydb.topic.reader.session.errors`` counts failed sessions or connection attempts.
+ The ``retry_decision`` attribute is ``retry`` when the Reader will reconnect and
+ ``stop`` when it terminates. ``status_code`` identifies the YDB status or exception
+ class, while ``error.type`` classifies transport, YDB, and other failures. Normal
+ Reader shutdown is not counted. This Reader-level metric has no ``topic`` attribute.
+* ``ydb.topic.reader.credit_balance_bytes`` reports the protocol byte credit granted to
+ the server and not yet consumed by read responses. It is ``0`` between stream
+ sessions, uses unit ``By``, and has the Reader attributes but no ``topic``.
+* ``ydb.topic.reader.local_buffer.messages`` reports the current number of messages
+ retained by the logical Reader, including messages waiting for decoding and decoded
+ messages waiting for the application. It has the Reader attributes but no ``topic``.
+* ``ydb.topic.reader.local_buffer.message_age.max`` reports, in seconds, the age of the
+ oldest retained message. It uses a monotonic clock, becomes ``0`` when the buffer is
+ empty, and has the Reader attributes but no ``topic``.
+* ``ydb.topic.reader.partition_session.count`` reports active partition sessions per
+ ``topic``. Configured topics report ``0`` before the first session starts. The metric
+ has the Reader attributes and ``topic``, but deliberately has no ``partition``
+ attribute.
+
+If ``reader_name`` is omitted or empty, the Reader still works and receives its normal
+process-local diagnostic name, but these thirteen metrics are not published. The runnable
+``examples/opentelemetry/otel_example.py`` creates a topic, writes one message, reads it
+with an explicitly named Reader, and exports its Reader metrics through OTLP.
+
+
Instrumented Spans and Attributes
---------------------------------
@@ -226,9 +405,11 @@ Running the Examples
--------------------
The runnable script is ``examples/opentelemetry/otel_example.py`` (bank table + concurrent
-Serializable transactions and ``app_startup`` / ``example_tli`` application spans). **Start
-Docker (YDB or the full stack) first**, then install and run on the host — see
-``examples/opentelemetry/README.md`` for the full order of commands and environment variables.
+Serializable transactions, ``app_startup`` / ``example_tli`` application spans, and an
+explicitly named topic Reader that emits receive, delivery, and buffer metrics). **Start Docker
+(YDB or the full stack) first**, then install and run on the host — see
+``examples/opentelemetry/README.md`` for the full order of commands and environment
+variables.
**Full stack in one command** (YDB + OTLP + Tempo + Grafana; the ``otel-example`` service is built from ``examples/opentelemetry/Dockerfile`` and runs the script once):
diff --git a/docs/topic.rst b/docs/topic.rst
index 4bf3a63a..f71187e2 100644
--- a/docs/topic.rst
+++ b/docs/topic.rst
@@ -183,12 +183,15 @@ Writer Parameters
codec=ydb.TopicCodec.GZIP, # Compress messages. Default: RAW (no compression).
auto_seqno=True, # Auto-increment sequence numbers (default: True).
auto_created_at=True, # Auto-set message timestamps (default: True).
+ writer_name="payments-writer", # Stable name that enables Writer metrics.
)
* ``producer_id`` is used for deduplication: if the same ``(producer_id, seqno)`` pair is
received again, YDB silently skips it.
* When ``auto_seqno=True`` the SDK assigns monotonically increasing sequence numbers.
Set ``auto_seqno=False`` and provide ``seqno`` manually when you need deterministic IDs.
+* Pass a stable, non-empty ``writer_name`` to publish Writer metrics. Omitting it does
+ not change Writer behavior, but disables its metrics.
Sending Messages
@@ -381,8 +384,16 @@ Reader Parameters
consumer="my-consumer",
buffer_size_bytes=50 * 1024 * 1024, # client-side buffer (default: 50 MB)
buffer_release_threshold=0.5, # see below (default: 0.5)
+ reader_name="payments-worker", # optional diagnostic name
)
+``reader_name`` identifies a logical reader in server-side diagnostics and SDK logs. If it is
+omitted or empty, the SDK generates a process-local name in the form ``reader-N``. The name
+remains unchanged when the reader reconnects. Explicit names do not have to be unique. Topic
+Reader metrics are published only when the application supplies a
+non-empty ``reader_name``. See :ref:`topic-reader-receive-metrics` for their semantics
+and attributes.
+
``buffer_size_bytes`` controls how many bytes the server is allowed to send before the client
signals that it is ready for more. The server will not exceed this limit.
diff --git a/examples/opentelemetry/Dockerfile b/examples/opentelemetry/Dockerfile
index 041eb4ab..961f5f32 100644
--- a/examples/opentelemetry/Dockerfile
+++ b/examples/opentelemetry/Dockerfile
@@ -20,3 +20,4 @@ RUN pip install --no-cache-dir -e '.[opentelemetry]' -r examples/opentelemetry/r
# Demo scripts.
COPY examples/opentelemetry/otel_example.py ./examples/opentelemetry/otel_example.py
COPY examples/opentelemetry/load_tank.py ./examples/opentelemetry/load_tank.py
+COPY examples/opentelemetry/verify_metrics.py ./examples/opentelemetry/verify_metrics.py
diff --git a/examples/opentelemetry/README.md b/examples/opentelemetry/README.md
index 33fd6899..47593d54 100644
--- a/examples/opentelemetry/README.md
+++ b/examples/opentelemetry/README.md
@@ -2,7 +2,71 @@
Async demo in [`otel_example.py`](otel_example.py): OTLP export, `enable_tracing()`,
`enable_metrics()`, `app_startup` and `example_tli` application spans, SDK client
-metrics, bank table, Serializable transactions (TLI-style load).
+metrics, bank table, Serializable transactions (TLI-style load), and an explicitly
+named topic Reader that emits `ydb.topic.reader.received.messages`,
+`ydb.topic.reader.received.bytes`, `ydb.topic.reader.delivered.messages`,
+`ydb.topic.reader.compression.duration`,
+`ydb.topic.reader.message.processing.duration`,
+`ydb.topic.reader.commit.queued`,
+`ydb.topic.reader.commit.acknowledged`,
+`ydb.topic.reader.commit_offset.lag.max`,
+`ydb.topic.reader.session.errors`,
+`ydb.topic.reader.credit_balance_bytes`,
+`ydb.topic.reader.local_buffer.messages`,
+`ydb.topic.reader.local_buffer.message_age.max`, and
+`ydb.topic.reader.partition_session.count`.
+
+## Topic Reader metrics in the example
+
+The demo enables metrics before creating the Reader and supplies
+`reader_name="otel-example-reader"`. It creates a unique topic for the run, deletes only
+that topic during cleanup, then writes and reads one message, producing:
+
+- `ydb.topic.reader.received.messages`: the number of messages received before
+ decoding, split by `topic`;
+- `ydb.topic.reader.received.bytes`: the total `ReadResponse.bytes_size` received by
+ that logical Reader, without a `topic` label because one response can contain data
+ from several topics;
+- `ydb.topic.reader.delivered.messages`: the number of messages returned by
+ `receive_message`, `receive_batch`, or `receive_batch_with_tx`, split by `topic`;
+- `ydb.topic.reader.compression.duration`: seconds spent decompressing each non-RAW
+ message, excluding decoder-executor queueing, split by `topic`, `codec`, and
+ `outcome`;
+- `ydb.topic.reader.message.processing.duration`: seconds from receipt until each
+ non-RAW message's batch is ready for delivery, including decoder queueing and split
+ by `topic`, `codec`, and `outcome`;
+- `ydb.topic.reader.commit.queued`: messages in commit ranges accepted by the SDK,
+ counted as `range.end - range.start` and split by `topic`;
+- `ydb.topic.reader.commit.acknowledged`: monotonic committed-offset advances reported
+ by regular acknowledgements, plus ranges from successful transaction commits, split
+ by `topic`;
+- `ydb.topic.reader.commit_offset.lag.max`: the largest gap between a regular stream
+ commit request's end offset and the acknowledged offset across active partitions,
+ split by `topic`;
+- `ydb.topic.reader.session.errors`: failed sessions or connection attempts, split by
+ `retry_decision=retry|stop`, `status_code`, and `error.type`; normal shutdown is not
+ counted and `topic` is omitted;
+- `ydb.topic.reader.credit_balance_bytes`: protocol byte credit granted to the server
+ and not yet consumed by read responses; `0` between stream sessions and without
+ `topic`;
+- `ydb.topic.reader.local_buffer.messages`: the current number of messages retained by
+ the logical Reader, including decoding and ready-to-deliver queues, without `topic`;
+- `ydb.topic.reader.local_buffer.message_age.max`: the age in seconds of the oldest
+ retained message, or `0` when the buffer is empty, without `topic`;
+- `ydb.topic.reader.partition_session.count`: the current number of active partition
+ sessions, split by `topic` and without a `partition` label; configured topics report
+ `0` before their first session starts.
+
+All thirteen metrics also carry `endpoint`, `database`, and `reader.name`; `consumer` is
+included for consumer-based reads. They remain attached to the same logical Reader
+across stream reconnects. Use the rate of the cumulative counters to compare incoming
+traffic with messages visible to the application, and read the observable instruments
+as current values.
+
+The example keeps the message buffered, then keeps the empty Reader open, for
+`OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS` seconds in each state. Its default is 7 seconds,
+which is longer than this stack's 5-second Prometheus scrape interval, so both the
+positive and zero gauge samples are observable.
[`load_tank.py`](load_tank.py) runs a small step-like load profile for the
metrics dashboard:
@@ -11,6 +75,15 @@ metrics dashboard:
Peak RPS -> Medium RPS -> Min RPS -> Medium RPS -> repeat
```
+The same process also creates a temporary multi-partition topic and keeps a named
+Reader and writer active for the whole run. By default the writer produces 20 messages
+per second while the Reader deliberately pauses for 0.1 seconds per message. This
+creates a visible, bounded SDK-side prefetch buffer and continuously exercises all
+Topic Reader counters, histograms, and gauges. A short Reader against a deliberately missing topic
+emits the deterministic `session.errors{retry_decision="stop"}` case without
+interrupting the main stream. A synthetic transient `ABORTED` probe exercises real
+multi-attempt retry metrics without changing the database.
+
Most steps assume the **repository root** as the current directory; the install step also shows the variant from this folder.
## 1. Start YDB (or the full stack) with Docker **first**
@@ -47,12 +120,118 @@ The first run builds the `otel-example` image from the local SDK source (`Docker
Grafana: http://localhost:3000
Prometheus: http://localhost:9090
+To turn this into an automated e2e check, start the stack in the background and run
+the verifier. It waits up to three minutes for every SDK metric, including the topic
+error probe, to carry the expected non-zero activity in Prometheus. The retry check
+also requires at least one operation with more than one attempt. The verifier reads
+the current load-generator run ID from the shared Compose volume and only accepts
+series carrying that `service.instance.id`, so metrics left by an older run cannot
+make the check pass:
+
+```sh
+docker compose -f examples/opentelemetry/compose-e2e.yaml up --build -d
+docker compose -f examples/opentelemetry/compose-e2e.yaml --profile verify run --rm metrics-verifier
+```
+
+The verifier returns exit code `1` and prints the missing Prometheus series if the
+contract is incomplete. Run it directly against another Prometheus instance with:
+
+```sh
+python examples/opentelemetry/verify_metrics.py \
+ --prometheus-url http://localhost:9090 \
+ --service-instance-id \
+ --require-error-probe
+```
+
+Use the value printed as `run_id` by `load_tank.py`, or set `LOAD_TANK_RUN_ID`
+explicitly before starting it. The verifier requires either `--service-instance-id`
+or `--run-id-file`; it never searches unscoped historical series.
+
Grafana is provisioned with the **YDB Python SDK Metrics** dashboard. It uses
-Prometheus queries for SDK metrics such as `db_client_operation_duration`,
+Prometheus queries for SDK metrics such as `ydb_client_operation_duration`,
`ydb_client_operation_failed`, `ydb_query_session_count`,
`ydb_query_session_pending_requests`, `ydb_query_session_create_time`, and
-`ydb_client_retry_duration`. Use Grafana Explore for ad-hoc traces through Tempo
-and metrics through Prometheus.
+`ydb_client_retry_duration`. The example also publishes the topic Reader received and
+delivered message counters by topic, the reader-level received-byte counter, and the
+reader-level local-buffer count and oldest-age gauges. It also publishes commit-queued
+and commit-acknowledged messages, commit-offset lag, and the active partition-session
+count by topic, the reader-level credit balance, and message processing/decompression
+latency histograms. Use Grafana Explore for ad-hoc traces through Tempo and metrics
+through Prometheus.
+
+In Grafana Explore, select the Prometheus data source and query:
+
+```promql
+ydb_topic_reader_local_buffer_messages{reader_name="otel-example-reader"}
+```
+
+The oldest buffered-message age is available as:
+
+```promql
+ydb_topic_reader_local_buffer_message_age_max_seconds{reader_name="otel-example-reader"}
+```
+
+Active partition sessions are available as:
+
+```promql
+ydb_topic_reader_partition_session_count{reader_name="otel-example-reader"}
+```
+
+The current byte credit granted to the server is available as:
+
+```promql
+ydb_topic_reader_credit_balance_bytes{reader_name="otel-example-reader"}
+```
+
+The six counters use Prometheus' conventional `_total` suffix, for example:
+
+```promql
+ydb_topic_reader_received_messages_total{reader_name="otel-example-reader"}
+```
+
+Commit ranges accepted by the Reader are available as:
+
+```promql
+ydb_topic_reader_commit_queued_total{reader_name="otel-example-reader"}
+```
+
+Successfully acknowledged commit ranges are available as:
+
+```promql
+ydb_topic_reader_commit_acknowledged_total{reader_name="otel-example-reader"}
+```
+
+The maximum outstanding commit-offset gap is available as:
+
+```promql
+ydb_topic_reader_commit_offset_lag_max{reader_name="otel-example-reader"}
+```
+
+The p95 time from message receipt until the decoded batch is ready is available as:
+
+```promql
+histogram_quantile(0.95,
+ sum by (le, reader_name, topic, codec, outcome) (
+ rate(ydb_topic_reader_message_processing_duration_seconds_bucket[$__rate_interval])
+ )
+)
+```
+
+The p95 time spent inside the decompressor is available as:
+
+```promql
+histogram_quantile(0.95,
+ sum by (le, reader_name, topic, codec, outcome) (
+ rate(ydb_topic_reader_compression_duration_seconds_bucket[$__rate_interval])
+ )
+)
+```
+
+Session errors appear only after a failed session or connection attempt:
+
+```promql
+ydb_topic_reader_session_errors_total{reader_name="load-tank-topic-error-probe", retry_decision="stop"}
+```
The SDK configures explicit OpenTelemetry histogram bucket boundaries for its
own duration and retry-attempt metrics. Duration values are recorded in seconds,
@@ -110,7 +289,13 @@ python examples/opentelemetry/load_tank.py
| `YDB_DATABASE` | default `/local` |
| `OTEL_EXPORTER_OTLP_ENDPOINT` | e.g. `http://otel-collector:4317` |
| `OTEL_SERVICE_NAME` | service name attached to exported telemetry |
-| `LOAD_TANK_TOTAL_TIME` | total load duration in seconds, default `6000` |
+| `LOAD_TANK_RUN_ID` | optional stable `service.instance.id`; a random ID is used for each run |
+| `LOAD_TANK_RUN_ID_FILE` | optional file to which the load tank atomically publishes its run ID |
+| `LOAD_TANK_READY_FILE` | optional readiness marker created after YDB and the error probe succeed |
+| `METRICS_SERVICE_INSTANCE_ID` | run ID selected by `verify_metrics.py` |
+| `METRICS_RUN_ID_FILE` | file from which `verify_metrics.py` reads the selected run ID |
+| `OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS` | seconds to expose each buffer-gauge state, default `7` |
+| `LOAD_TANK_TOTAL_TIME` | total load duration in seconds, default `300`; Compose uses `6000` |
| `LOAD_TANK_WORKERS` | number of concurrent workers, default `40` |
| `LOAD_TANK_POOL_SIZE` | query session pool size, default `20` |
| `LOAD_TANK_PEAK_RPS` | peak phase target RPS, default `120` |
@@ -123,8 +308,57 @@ python examples/opentelemetry/load_tank.py
| `LOAD_TANK_PRESSURE_ACQUIRE_TIMEOUT` | short acquire timeout for timeout metrics, default `1.0` |
| `LOAD_TANK_PRESSURE_INTERVAL` | pause between pressure rounds, default `0.2` |
| `LOAD_TANK_SESSION_CHURN_INTERVAL` | interval for creating fresh sessions, default `2.0` |
+| `LOAD_TANK_RETRY_PROBE_INTERVAL` | interval between deterministic retry probes, default `5.0`; set `0` to disable |
+| `LOAD_TANK_RETRY_PROBE_FAILURES` | transient `ABORTED` attempts before success, default `2` |
+| `LOAD_TANK_TOPIC_WRITE_RPS` | topic producer rate, default `20`; set `0` to disable topic load |
+| `LOAD_TANK_TOPIC_READ_DELAY` | delay after each topic message, default `0.1` seconds |
+| `LOAD_TANK_TOPIC_MESSAGE_SIZE` | topic payload size, default `1024` bytes |
+| `LOAD_TANK_TOPIC_PARTITIONS` | temporary topic partition count, default `2` |
+| `LOAD_TANK_TOPIC_READER_NAME` | stable Reader metric label, default `load-tank-topic-reader` |
+| `LOAD_TANK_TOPIC_ERROR_PROBE_ENDPOINT` | endpoint used for a missing-topic error probe; defaults to `YDB_ENDPOINT` |
+| `LOAD_TANK_TOPIC_ERROR_PROBE_SECONDS` | probe duration, default `0`; Compose sets it to `5` |
| `LOAD_TANK_PEAK_DURATION` | peak phase duration in seconds, default `60` |
| `LOAD_TANK_MEDIUM_DURATION` | medium phase duration in seconds, default `90` |
| `LOAD_TANK_MIN_DURATION` | low phase duration in seconds, default `60` |
| `LOAD_TANK_QUERY` | query executed by workers, default `SELECT 1 AS value` |
| `LOAD_TANK_ERROR_QUERY` | query used to produce failed-operation metrics |
+
+## Load-test recipes
+
+Run only the infrastructure first, then start one or more isolated load-generator
+containers with overrides. This makes test duration and intensity explicit and keeps
+Grafana available after a run:
+
+```sh
+docker compose -f examples/opentelemetry/compose-e2e.yaml up -d \
+ ydb otel-collector prometheus tempo grafana
+
+# Five-minute smoke/load run.
+docker compose -f examples/opentelemetry/compose-e2e.yaml run --rm \
+ -e LOAD_TANK_TOTAL_TIME=300 \
+ -e LOAD_TANK_PEAK_RPS=200 \
+ -e LOAD_TANK_WORKERS=80 \
+ load-generator
+```
+
+Useful scenarios:
+
+- **Throughput staircase:** increase `PEAK_RPS` until achieved RPS stops following the
+ target or p95/p99 latency bends sharply.
+- **Pool saturation:** keep `LOAD_TANK_PRESSURE_POOL_SIZE=1`, increase
+ `LOAD_TANK_PRESSURE_WORKERS`, and watch pending requests and acquire timeouts.
+- **Retries:** increase `LOAD_TANK_RETRY_PROBE_FAILURES` and watch retry-attempt count,
+ average attempts, retrying calls, and retry duration.
+- **Topic backpressure:** make `1 / LOAD_TANK_TOPIC_READ_DELAY` lower than
+ `LOAD_TANK_TOPIC_WRITE_RPS`; the local-buffer count and oldest-message age should
+ rise while received and delivered rates diverge.
+- **Balanced topic stream:** set read capacity above write RPS; the buffer and age
+ should return close to zero while received and delivered rates converge.
+- **Reconnect/chaos:** use the repository's `tests/slo` multi-node chaos setup for a
+ long run. The Reader session-error counter should grow while message delivery
+ resumes; do not restart this example's single in-memory YDB merely to create an
+ error sample.
+
+For repeatable comparison, change one dimension at a time, warm up before measuring,
+keep a fixed measurement window, and record offered RPS, achieved RPS, p50/p95/p99,
+error rate, retry ratio, pool saturation, and topic receive/delivery lag.
diff --git a/examples/opentelemetry/compose-e2e.yaml b/examples/opentelemetry/compose-e2e.yaml
index e0bf9a5e..59f46a6f 100644
--- a/examples/opentelemetry/compose-e2e.yaml
+++ b/examples/opentelemetry/compose-e2e.yaml
@@ -12,6 +12,7 @@ services:
image: ydbplatform/local-ydb:trunk
restart: always
platform: linux/amd64
+ hostname: localhost
environment:
YDB_DEFAULT_LOG_LEVEL: NOTICE
GRPC_TLS_PORT: "2135"
@@ -84,6 +85,7 @@ services:
YDB_DATABASE: /local
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317
OTEL_SERVICE_NAME: ydb-otel-example
+ OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS: "7"
depends_on:
ydb:
condition: service_healthy
@@ -95,12 +97,19 @@ services:
build:
context: ../..
dockerfile: examples/opentelemetry/Dockerfile
- command: ["python", "examples/opentelemetry/load_tank.py"]
+ command:
+ [
+ "sh",
+ "-c",
+ "rm -f /tmp/ydb-load-tank-ready && exec python examples/opentelemetry/load_tank.py",
+ ]
environment:
YDB_ENDPOINT: grpc://ydb:2136
YDB_DATABASE: /local
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317
OTEL_SERVICE_NAME: ydb-python-load-tank
+ LOAD_TANK_RUN_ID_FILE: /run/ydb-metrics/run-id
+ LOAD_TANK_READY_FILE: /tmp/ydb-load-tank-ready
LOAD_TANK_TOTAL_TIME: "6000"
LOAD_TANK_WORKERS: "40"
LOAD_TANK_POOL_SIZE: "20"
@@ -114,9 +123,51 @@ services:
LOAD_TANK_PRESSURE_ACQUIRE_TIMEOUT: "1.0"
LOAD_TANK_PRESSURE_INTERVAL: "0.2"
LOAD_TANK_SESSION_CHURN_INTERVAL: "2.0"
+ LOAD_TANK_RETRY_PROBE_INTERVAL: "5.0"
+ LOAD_TANK_RETRY_PROBE_FAILURES: "2"
+ LOAD_TANK_TOPIC_WRITE_RPS: "20"
+ LOAD_TANK_TOPIC_READ_DELAY: "0.1"
+ LOAD_TANK_TOPIC_MESSAGE_SIZE: "1024"
+ LOAD_TANK_TOPIC_PARTITIONS: "2"
+ LOAD_TANK_TOPIC_ERROR_PROBE_ENDPOINT: grpc://ydb:2136
+ LOAD_TANK_TOPIC_ERROR_PROBE_SECONDS: "5"
+ volumes:
+ - metrics-run:/run/ydb-metrics
depends_on:
ydb:
condition: service_healthy
otel-collector:
condition: service_started
+ healthcheck:
+ test: ["CMD-SHELL", "test -f /tmp/ydb-load-tank-ready"]
+ interval: 1s
+ timeout: 1s
+ retries: 90
restart: "no"
+
+ metrics-verifier:
+ profiles: ["verify"]
+ build:
+ context: ../..
+ dockerfile: examples/opentelemetry/Dockerfile
+ command:
+ [
+ "python",
+ "examples/opentelemetry/verify_metrics.py",
+ "--prometheus-url",
+ "http://prometheus:9090",
+ "--require-error-probe",
+ ]
+ environment:
+ METRICS_RUN_ID_FILE: /run/ydb-metrics/run-id
+ volumes:
+ - metrics-run:/run/ydb-metrics:ro
+ depends_on:
+ prometheus:
+ condition: service_started
+ load-generator:
+ condition: service_healthy
+ restart: "no"
+
+volumes:
+ metrics-run:
diff --git a/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json b/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json
index 1a0706aa..6d612101 100644
--- a/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json
+++ b/examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json
@@ -14,7 +14,7 @@
"options": { "legend": { "displayMode": "table", "placement": "right" } },
"targets": [
{
- "expr": "sum by (ydb_operation_name) (rate(db_client_operation_duration_seconds_count[$__rate_interval]))",
+ "expr": "sum by (ydb_operation_name) (rate(ydb_client_operation_duration_seconds_count[$__rate_interval]))",
"legendFormat": "{{ydb_operation_name}}",
"datasource": { "type": "prometheus", "uid": "prometheus" }
}
@@ -99,17 +99,17 @@
"options": { "legend": { "displayMode": "table", "placement": "right" } },
"targets": [
{
- "expr": "histogram_quantile(0.50, sum by (le, ydb_operation_name) (rate(db_client_operation_duration_seconds_bucket[$__rate_interval])))",
+ "expr": "histogram_quantile(0.50, sum by (le, ydb_operation_name) (rate(ydb_client_operation_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p50 - {{ydb_operation_name}}",
"datasource": { "type": "prometheus", "uid": "prometheus" }
},
{
- "expr": "histogram_quantile(0.95, sum by (le, ydb_operation_name) (rate(db_client_operation_duration_seconds_bucket[$__rate_interval])))",
+ "expr": "histogram_quantile(0.95, sum by (le, ydb_operation_name) (rate(ydb_client_operation_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p95 - {{ydb_operation_name}}",
"datasource": { "type": "prometheus", "uid": "prometheus" }
},
{
- "expr": "histogram_quantile(0.99, sum by (le, ydb_operation_name) (rate(db_client_operation_duration_seconds_bucket[$__rate_interval])))",
+ "expr": "histogram_quantile(0.99, sum by (le, ydb_operation_name) (rate(ydb_client_operation_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p99 - {{ydb_operation_name}}",
"datasource": { "type": "prometheus", "uid": "prometheus" }
}
@@ -186,6 +186,175 @@
"datasource": { "type": "prometheus", "uid": "prometheus" }
}
]
+ },
+ {
+ "title": "Closed Query Sessions",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
+ "fieldConfig": { "defaults": { "unit": "short" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "sum by (ydb_query_session_pool_name, reason) (increase(ydb_query_session_closed_total[$__rate_interval]))",
+ "legendFormat": "{{ydb_query_session_pool_name}} - {{reason}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Reader Session Errors",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
+ "fieldConfig": { "defaults": { "unit": "short" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, retry_decision) (increase(ydb_topic_reader_session_errors_total[$__rate_interval]))",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{retry_decision}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Message Rate",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+ "fieldConfig": { "defaults": { "unit": "msgps" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_received_messages_total[$__rate_interval]))",
+ "legendFormat": "received - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ },
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_delivered_messages_total[$__rate_interval]))",
+ "legendFormat": "delivered - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ },
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_commit_queued_total[$__rate_interval]))",
+ "legendFormat": "commit queued - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ },
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (rate(ydb_topic_reader_commit_acknowledged_total[$__rate_interval]))",
+ "legendFormat": "commit acknowledged - {{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Received Throughput",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+ "fieldConfig": { "defaults": { "unit": "Bps" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name) (rate(ydb_topic_reader_received_bytes_total[$__rate_interval]))",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Reader Local Buffer",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 8, "x": 0, "y": 48 },
+ "fieldConfig": { "defaults": { "unit": "short" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name) (ydb_topic_reader_local_buffer_messages)",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Oldest Buffered Topic Message",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 8, "x": 8, "y": 48 },
+ "fieldConfig": { "defaults": { "unit": "s" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "max by (service_instance_id, endpoint, database, consumer, reader_name) (ydb_topic_reader_local_buffer_message_age_max_seconds)",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Partition Sessions",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 8, "x": 16, "y": 48 },
+ "fieldConfig": { "defaults": { "unit": "short" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name, topic) (ydb_topic_reader_partition_session_count)",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Commit Offset Lag",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 },
+ "fieldConfig": { "defaults": { "unit": "short" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "max by (service_instance_id, endpoint, database, consumer, reader_name, topic) (ydb_topic_reader_commit_offset_lag_max)",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Reader Credit Balance",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 },
+ "fieldConfig": { "defaults": { "unit": "bytes" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "sum by (service_instance_id, endpoint, database, consumer, reader_name) (ydb_topic_reader_credit_balance_bytes)",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Message Processing p95",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 },
+ "fieldConfig": { "defaults": { "unit": "s" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, sum by (le, service_instance_id, endpoint, database, consumer, reader_name, topic, codec, outcome) (rate(ydb_topic_reader_message_processing_duration_seconds_bucket[$__rate_interval])))",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}} - {{codec}} - {{outcome}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
+ },
+ {
+ "title": "Topic Decompression p95",
+ "type": "timeseries",
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 },
+ "fieldConfig": { "defaults": { "unit": "s" } },
+ "options": { "legend": { "displayMode": "table", "placement": "right" } },
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, sum by (le, service_instance_id, endpoint, database, consumer, reader_name, topic, codec, outcome) (rate(ydb_topic_reader_compression_duration_seconds_bucket[$__rate_interval])))",
+ "legendFormat": "{{service_instance_id}} - {{endpoint}} - {{database}} - {{consumer}} - {{reader_name}} - {{topic}} - {{codec}} - {{outcome}}",
+ "datasource": { "type": "prometheus", "uid": "prometheus" }
+ }
+ ]
}
]
}
diff --git a/examples/opentelemetry/load_tank.py b/examples/opentelemetry/load_tank.py
index 801ddadf..7fccbf13 100644
--- a/examples/opentelemetry/load_tank.py
+++ b/examples/opentelemetry/load_tank.py
@@ -6,13 +6,10 @@
import os
import random
import time
+import uuid
from dataclasses import dataclass
-from typing import AsyncIterator, Tuple
-
-from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
-from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics._internal.export import PeriodicExportingMetricReader
-from opentelemetry.sdk.resources import Resource
+from pathlib import Path
+from typing import Any, AsyncIterator, Awaitable, Dict, Optional, Tuple
import ydb
from ydb.opentelemetry import enable_metrics
@@ -24,6 +21,9 @@ class LoadConfig:
database: str
otlp_endpoint: str
service_name: str
+ run_id: str
+ run_id_file: str
+ ready_file: str
pool_size: int
worker_count: int
peak_rps: int
@@ -42,6 +42,15 @@ class LoadConfig:
pressure_acquire_timeout: float
pressure_interval: float
session_churn_interval: float
+ retry_probe_interval: float
+ retry_probe_failures: int
+ topic_write_rps: int
+ topic_read_delay: float
+ topic_message_size: int
+ topic_partitions: int
+ topic_reader_name: str
+ topic_error_probe_endpoint: str
+ topic_error_probe_seconds: float
def _env(name: str, default: str) -> str:
@@ -63,6 +72,9 @@ def _load_config() -> LoadConfig:
database=_env("YDB_DATABASE", "/local"),
otlp_endpoint=_env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317"),
service_name=_env("OTEL_SERVICE_NAME", "ydb-python-load-tank"),
+ run_id=_env("LOAD_TANK_RUN_ID", uuid.uuid4().hex),
+ run_id_file=_env("LOAD_TANK_RUN_ID_FILE", ""),
+ ready_file=_env("LOAD_TANK_READY_FILE", ""),
pool_size=_env_int("LOAD_TANK_POOL_SIZE", 20),
worker_count=_env_int("LOAD_TANK_WORKERS", 40),
peak_rps=_env_int("LOAD_TANK_PEAK_RPS", 120),
@@ -81,9 +93,94 @@ def _load_config() -> LoadConfig:
pressure_acquire_timeout=_env_float("LOAD_TANK_PRESSURE_ACQUIRE_TIMEOUT", 1.0),
pressure_interval=_env_float("LOAD_TANK_PRESSURE_INTERVAL", 0.2),
session_churn_interval=_env_float("LOAD_TANK_SESSION_CHURN_INTERVAL", 2.0),
+ retry_probe_interval=_env_float("LOAD_TANK_RETRY_PROBE_INTERVAL", 5.0),
+ retry_probe_failures=_env_int("LOAD_TANK_RETRY_PROBE_FAILURES", 2),
+ topic_write_rps=_env_int("LOAD_TANK_TOPIC_WRITE_RPS", 20),
+ topic_read_delay=_env_float("LOAD_TANK_TOPIC_READ_DELAY", 0.1),
+ topic_message_size=_env_int("LOAD_TANK_TOPIC_MESSAGE_SIZE", 1024),
+ topic_partitions=_env_int("LOAD_TANK_TOPIC_PARTITIONS", 2),
+ topic_reader_name=_env("LOAD_TANK_TOPIC_READER_NAME", "load-tank-topic-reader"),
+ topic_error_probe_endpoint=_env(
+ "LOAD_TANK_TOPIC_ERROR_PROBE_ENDPOINT",
+ _env("YDB_ENDPOINT", "grpc://localhost:2136"),
+ ),
+ topic_error_probe_seconds=_env_float("LOAD_TANK_TOPIC_ERROR_PROBE_SECONDS", 0),
)
+def _publish_run_id(run_id: str, run_id_file: str, ready_file: str) -> None:
+ if ready_file:
+ Path(ready_file).unlink(missing_ok=True)
+
+ if run_id_file:
+ target = Path(run_id_file)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ temporary = target.with_name("%s.%s.tmp" % (target.name, uuid.uuid4().hex))
+ temporary.write_text("%s\n" % run_id, encoding="utf-8")
+ os.replace(temporary, target)
+
+ if ready_file:
+ target = Path(ready_file)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.touch()
+
+
+async def _await_with_background(
+ awaitable: Awaitable[Any],
+ background_tasks: Dict[asyncio.Task, str],
+) -> Any:
+ operation_task = asyncio.ensure_future(awaitable)
+ try:
+ while True:
+ done, _ = await asyncio.wait(
+ {operation_task, *background_tasks},
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ for task in done:
+ if task is operation_task:
+ continue
+ name = background_tasks[task]
+ await _raise_background_failure(task, name)
+
+ if operation_task in done:
+ return await operation_task
+ finally:
+ if not operation_task.done():
+ operation_task.cancel()
+ await asyncio.gather(operation_task, return_exceptions=True)
+
+
+async def _raise_background_failure(task: asyncio.Task, name: str) -> None:
+ try:
+ await task
+ except asyncio.CancelledError as error:
+ raise RuntimeError("%s was cancelled unexpectedly" % name) from error
+ except Exception as error:
+ raise RuntimeError("%s failed" % name) from error
+ raise RuntimeError("%s stopped unexpectedly" % name)
+
+
+async def _cancel_background_tasks(
+ background_tasks: Dict[asyncio.Task, str],
+ excluded_task: Optional[asyncio.Task] = None,
+ raise_failures: bool = False,
+) -> None:
+ tasks = [(task, name) for task, name in background_tasks.items() if task is not excluded_task]
+ unexpectedly_done = [(task, name) for task, name in tasks if task.done()]
+
+ for task, _ in tasks:
+ task.cancel()
+ results = await asyncio.gather(*(task for task, _ in tasks), return_exceptions=True)
+
+ if not raise_failures:
+ return
+ if unexpectedly_done:
+ await _raise_background_failure(*unexpectedly_done[0])
+ for (task, name), result in zip(tasks, results):
+ if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError):
+ await _raise_background_failure(task, name)
+
+
async def _load_steps(config: LoadConfig) -> AsyncIterator[Tuple[int, str, int]]:
pattern = (
(config.peak_rps, "Peak", config.peak_duration),
@@ -146,7 +243,7 @@ async def _error_worker(pool: ydb.aio.QuerySessionPool, config: LoadConfig, stop
while not stop.is_set():
try:
await pool.execute_with_retries(config.error_query)
- except Exception:
+ except ydb.SchemeError:
pass
next_tick += interval
@@ -169,13 +266,13 @@ async def contender() -> None:
try:
async with pool.checkout(timeout=config.pressure_acquire_timeout):
pass
- except Exception:
+ except ydb.SessionPoolEmpty:
pass
holder_task = asyncio.create_task(holder())
await asyncio.sleep(0)
contenders = [asyncio.create_task(contender()) for _ in range(config.pressure_workers)]
- await asyncio.gather(holder_task, *contenders, return_exceptions=True)
+ await asyncio.gather(holder_task, *contenders)
async def _pool_pressure_worker(driver: ydb.aio.Driver, config: LoadConfig, stop: asyncio.Event) -> None:
@@ -209,10 +306,147 @@ async def _session_churn_worker(driver: ydb.aio.Driver, config: LoadConfig, stop
pass
+async def _retry_probe_worker(config: LoadConfig, stop: asyncio.Event) -> None:
+ if config.retry_probe_interval <= 0 or config.retry_probe_failures <= 0:
+ return
+
+ retry_settings = ydb.RetrySettings(max_retries=config.retry_probe_failures)
+ while not stop.is_set():
+ attempt = 0
+
+ async def transient_failure() -> None:
+ nonlocal attempt
+ attempt += 1
+ if attempt <= config.retry_probe_failures:
+ raise ydb.Aborted("load-tank retry probe")
+
+ await ydb.retry_operation_async(transient_failure, retry_settings)
+ try:
+ await asyncio.wait_for(stop.wait(), timeout=config.retry_probe_interval)
+ except asyncio.TimeoutError:
+ pass
+
+
+def _topic_payload(sequence_number: int, size: int) -> bytes:
+ prefix = "%s:%s" % (sequence_number, time.time_ns())
+ return prefix.encode("ascii").ljust(max(size, 1), b"x")[: max(size, 1)]
+
+
+async def _topic_writer(writer, config: LoadConfig, stop: asyncio.Event) -> None:
+ interval = 1.0 / config.topic_write_rps
+ next_tick = time.monotonic()
+ sequence_number = 0
+
+ while not stop.is_set():
+ await writer.write(_topic_payload(sequence_number, config.topic_message_size))
+ sequence_number += 1
+ next_tick += interval
+ delay = next_tick - time.monotonic()
+ if delay > 0:
+ try:
+ await asyncio.wait_for(stop.wait(), timeout=delay)
+ except asyncio.TimeoutError:
+ pass
+ else:
+ await asyncio.sleep(0)
+
+
+async def _topic_reader(reader, config: LoadConfig, stop: asyncio.Event) -> None:
+ while not stop.is_set():
+ try:
+ message = await asyncio.wait_for(reader.receive_message(), timeout=1)
+ except asyncio.TimeoutError:
+ continue
+
+ if config.topic_read_delay > 0:
+ try:
+ await asyncio.wait_for(stop.wait(), timeout=config.topic_read_delay)
+ except asyncio.TimeoutError:
+ pass
+ await reader.commit_with_ack(message)
+
+
+async def _topic_stream_worker(driver: ydb.aio.Driver, config: LoadConfig, stop: asyncio.Event) -> None:
+ if config.topic_write_rps <= 0:
+ return
+
+ topic_path = "%s/otel-load-%s" % (config.database.rstrip("/"), uuid.uuid4().hex)
+ consumer = "load-tank-consumer"
+ await driver.topic_client.create_topic(
+ topic_path,
+ min_active_partitions=config.topic_partitions,
+ partition_count_limit=config.topic_partitions,
+ consumers=[consumer],
+ )
+ print(
+ "Topic stream: path=%s writer=%s RPS reader_delay=%ss message_size=%sB partitions=%s"
+ % (
+ topic_path,
+ config.topic_write_rps,
+ config.topic_read_delay,
+ config.topic_message_size,
+ config.topic_partitions,
+ )
+ )
+
+ try:
+ async with driver.topic_client.writer(topic_path, codec=ydb.TopicCodec.GZIP) as writer:
+ async with driver.topic_client.reader(
+ topic_path,
+ consumer,
+ reader_name=config.topic_reader_name,
+ ) as reader:
+ writer_task = asyncio.create_task(_topic_writer(writer, config, stop))
+ reader_task = asyncio.create_task(_topic_reader(reader, config, stop))
+ try:
+ await asyncio.gather(writer_task, reader_task)
+ finally:
+ writer_task.cancel()
+ reader_task.cancel()
+ await asyncio.gather(writer_task, reader_task, return_exceptions=True)
+ finally:
+ await driver.topic_client.drop_topic(topic_path)
+
+
+async def _topic_error_probe(config: LoadConfig) -> None:
+ if config.topic_error_probe_seconds <= 0:
+ return
+
+ topic_path = "%s/otel-unreachable" % config.database.rstrip("/")
+ try:
+ async with ydb.aio.Driver(
+ endpoint=config.topic_error_probe_endpoint,
+ database=config.database,
+ disable_discovery=True,
+ ) as driver:
+ async with driver.topic_client.reader(
+ topic_path,
+ "load-tank-error-probe",
+ reader_name="load-tank-topic-error-probe",
+ ) as reader:
+ await asyncio.wait_for(reader.wait_message(), timeout=config.topic_error_probe_seconds)
+ except ydb.SchemeError as error:
+ print("Topic error probe produced the expected session failure: %s" % type(error).__name__)
+ except asyncio.TimeoutError as error:
+ raise RuntimeError("Topic error probe did not produce SCHEME_ERROR before its timeout") from error
+ else:
+ raise RuntimeError("Topic error probe unexpectedly received a message")
+
+
async def main() -> None:
+ from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+ from opentelemetry.sdk.metrics import MeterProvider
+ from opentelemetry.sdk.metrics._internal.export import PeriodicExportingMetricReader
+ from opentelemetry.sdk.resources import Resource
+
config = _load_config()
- resource = Resource(attributes={"service.name": config.service_name})
+ resource = Resource(
+ attributes={
+ "service.name": config.service_name,
+ "service.instance.id": config.run_id,
+ }
+ )
metric_reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=config.otlp_endpoint),
export_interval_millis=2000,
@@ -222,11 +456,15 @@ async def main() -> None:
print(
"=== YDB Python SDK load tank ===\n"
- " total=%ss workers=%s pool_size=%s query=%r error_rps=%s\n"
+ " run_id=%s total=%ss workers=%s pool_size=%s query=%r error_rps=%s\n"
" pressure_pool_size=%s pressure_workers=%s pressure_timeout=%ss session_churn_interval=%ss\n"
+ " retry_probe_interval=%ss retry_probe_failures=%s\n"
+ " topic_write_rps=%s topic_read_delay=%ss topic_message_size=%sB topic_partitions=%s\n"
+ " topic_error_probe=%ss at %s\n"
" pattern: Peak(%s rps, %ss) -> Medium(%s rps, %ss) -> "
"Min(%s rps, %ss) -> Medium -> repeat"
% (
+ config.run_id,
config.total_time,
config.worker_count,
config.pool_size,
@@ -236,6 +474,14 @@ async def main() -> None:
config.pressure_workers,
config.pressure_acquire_timeout,
config.session_churn_interval,
+ config.retry_probe_interval,
+ config.retry_probe_failures,
+ config.topic_write_rps,
+ config.topic_read_delay,
+ config.topic_message_size,
+ config.topic_partitions,
+ config.topic_error_probe_seconds,
+ config.topic_error_probe_endpoint,
config.peak_rps,
config.peak_duration,
config.medium_rps,
@@ -245,42 +491,85 @@ async def main() -> None:
)
)
- async with ydb.aio.Driver(
- endpoint=config.endpoint,
- database=config.database,
- disable_discovery=True,
- ) as driver:
- await driver.wait(timeout=60)
-
- async with ydb.aio.QuerySessionPool(driver, size=config.pool_size, name="load-tank") as pool:
- queue: asyncio.Queue[object] = asyncio.Queue(maxsize=max(config.worker_count * 4, config.peak_rps))
- stop = asyncio.Event()
- workers = [
- asyncio.create_task(_worker(pool, queue, config.query, stop)) for _ in range(config.worker_count)
- ]
- error_task = asyncio.create_task(_error_worker(pool, config, stop))
- pressure_task = asyncio.create_task(_pool_pressure_worker(driver, config, stop))
- churn_task = asyncio.create_task(_session_churn_worker(driver, config, stop))
-
- try:
- async for rps, label, duration in _load_steps(config):
- print("[%s] Phase: %s (%s RPS for %ss)" % (time.strftime("%H:%M:%S"), label, rps, duration))
- await _feed_phase(queue, rps, duration)
- await asyncio.sleep(random.random() / 10.0)
-
- await queue.join()
- finally:
- stop.set()
- for worker in workers:
- worker.cancel()
- error_task.cancel()
- pressure_task.cancel()
- churn_task.cancel()
- await asyncio.gather(*workers, error_task, pressure_task, churn_task, return_exceptions=True)
-
- print("Waiting 10s to flush metrics...")
- await asyncio.sleep(10)
- meter_provider.shutdown()
+ try:
+ async with ydb.aio.Driver(
+ endpoint=config.endpoint,
+ database=config.database,
+ disable_discovery=True,
+ ) as driver:
+ await driver.wait(timeout=60)
+ await _topic_error_probe(config)
+ _publish_run_id(config.run_id, config.run_id_file, config.ready_file)
+
+ async with ydb.aio.QuerySessionPool(driver, size=config.pool_size, name="load-tank") as pool:
+ queue: asyncio.Queue[object] = asyncio.Queue(maxsize=max(config.worker_count * 4, config.peak_rps))
+ stop = asyncio.Event()
+ background_tasks: Dict[asyncio.Task, str] = {}
+
+ def start(coroutine: Awaitable[None], name: str) -> asyncio.Task:
+ task = asyncio.create_task(coroutine)
+ background_tasks[task] = name
+ return task
+
+ for index in range(config.worker_count):
+ start(_worker(pool, queue, config.query, stop), "query worker %s" % index)
+ if config.error_rps > 0:
+ start(_error_worker(pool, config, stop), "query error worker")
+ if config.pressure_workers > 0 and config.pressure_pool_size > 0:
+ start(_pool_pressure_worker(driver, config, stop), "pool pressure worker")
+ if config.session_churn_interval > 0:
+ start(_session_churn_worker(driver, config, stop), "session churn worker")
+ if config.retry_probe_interval > 0 and config.retry_probe_failures > 0:
+ start(_retry_probe_worker(config, stop), "retry probe worker")
+ topic_task = None
+ if config.topic_write_rps > 0:
+ topic_task = start(_topic_stream_worker(driver, config, stop), "topic stream worker")
+
+ completed = False
+ try:
+ async for rps, label, duration in _load_steps(config):
+ print("[%s] Phase: %s (%s RPS for %ss)" % (time.strftime("%H:%M:%S"), label, rps, duration))
+ await _await_with_background(_feed_phase(queue, rps, duration), background_tasks)
+ await _await_with_background(asyncio.sleep(random.random() / 10.0), background_tasks)
+
+ await _await_with_background(queue.join(), background_tasks)
+ completed = True
+ finally:
+ topic_was_done = topic_task is not None and topic_task.done()
+ stop.set()
+ background_failure = None
+ try:
+ await _cancel_background_tasks(
+ background_tasks,
+ excluded_task=topic_task,
+ raise_failures=completed,
+ )
+ except Exception as error:
+ if not completed:
+ raise
+ background_failure = error
+
+ if topic_task is not None:
+ try:
+ await asyncio.wait_for(topic_task, timeout=30)
+ except asyncio.TimeoutError:
+ topic_task.cancel()
+ await asyncio.gather(topic_task, return_exceptions=True)
+ if completed:
+ raise RuntimeError("Topic load did not stop within 30 seconds")
+ except Exception:
+ if completed:
+ raise
+ else:
+ if completed and topic_was_done:
+ raise RuntimeError("Topic stream worker stopped unexpectedly")
+ if background_failure is not None:
+ raise background_failure
+
+ print("Waiting 10s to flush metrics...")
+ await asyncio.sleep(10)
+ finally:
+ meter_provider.shutdown()
if __name__ == "__main__":
diff --git a/examples/opentelemetry/otel_example.py b/examples/opentelemetry/otel_example.py
index 3bdd6a6b..61d06c11 100644
--- a/examples/opentelemetry/otel_example.py
+++ b/examples/opentelemetry/otel_example.py
@@ -1,4 +1,4 @@
-"""OpenTelemetry + YDB demo: bank table and concurrent transactions (TLI-style workload).
+"""OpenTelemetry + YDB demo: table operations, retries, and topic metrics.
Uses ``disable_discovery=True`` so a single ``grpc://…`` to local YDB (e.g. ``local-ydb`` in Docker
from the host) is not replaced by internal discovery addresses.
@@ -8,6 +8,7 @@
import asyncio
import os
+import uuid
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics import MeterProvider
@@ -44,10 +45,42 @@ async def _bank_read_update(tx) -> None:
pass
+async def _topic_reader_metrics_demo(
+ driver: ydb.aio.Driver,
+ database: str,
+ observation_seconds: float,
+) -> None:
+ topic_path = f"{database.rstrip('/')}/otel-reader-metrics-{uuid.uuid4().hex}"
+ consumer = "otel-example-consumer"
+
+ await driver.topic_client.create_topic(topic_path, consumers=[consumer])
+ try:
+ async with driver.topic_client.writer(
+ topic_path,
+ codec=ydb.TopicCodec.GZIP,
+ writer_name="otel-example-writer",
+ ) as writer:
+ await writer.write("OpenTelemetry topic Reader metrics")
+
+ async with driver.topic_client.reader(
+ topic_path,
+ consumer,
+ reader_name="otel-example-reader",
+ ) as reader:
+ await asyncio.wait_for(reader.wait_message(), timeout=10)
+ await asyncio.sleep(observation_seconds)
+ message = await asyncio.wait_for(reader.receive_message(), timeout=10)
+ await reader.commit_with_ack(message)
+ await asyncio.sleep(observation_seconds)
+ finally:
+ await driver.topic_client.drop_topic(topic_path)
+
+
async def main() -> None:
endpoint = _env("YDB_ENDPOINT", "grpc://localhost:2136")
database = _env("YDB_DATABASE", "/local")
otlp_endpoint = _env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317")
+ topic_buffer_observation_seconds = float(_env("OTEL_TOPIC_BUFFER_OBSERVATION_SECONDS", "7"))
resource = Resource(attributes={"service.name": _env("OTEL_SERVICE_NAME", "ydb-otel-example")})
tracer_provider = TracerProvider(resource=resource)
@@ -96,6 +129,9 @@ async def concurrent_task(task_num: int) -> None:
final_rows = await pool.execute_with_retries("SELECT amount FROM bank WHERE id = 1")
amount = int(list(final_rows[0].rows)[0]["amount"])
print(f"Final amount (after serializable retries): {amount}")
+
+ print("Emitting topic metrics...")
+ await _topic_reader_metrics_demo(driver, database, topic_buffer_observation_seconds)
print("Application will shut down in 15 seconds...")
await asyncio.sleep(15)
tracer_provider.shutdown()
diff --git a/examples/opentelemetry/verify_metrics.py b/examples/opentelemetry/verify_metrics.py
new file mode 100644
index 00000000..24b08628
--- /dev/null
+++ b/examples/opentelemetry/verify_metrics.py
@@ -0,0 +1,295 @@
+"""Verify that every YDB Python SDK metric reached Prometheus."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import time
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, Tuple
+
+
+@dataclass(frozen=True)
+class MetricCheck:
+ description: str
+ prometheus_name: str
+ promql: str = ""
+ requires_error_probe: bool = False
+
+
+LOOKBACK = "10m"
+
+
+def _prometheus_label_value(value: str) -> str:
+ return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+def _selector(**labels: str) -> str:
+ return "{%s}" % ",".join('%s="%s"' % (name, _prometheus_label_value(value)) for name, value in labels.items())
+
+
+def _present(prometheus_name: str, selector: str) -> str:
+ return "count(count_over_time(%s%s[%s])) > 0" % (prometheus_name, selector, LOOKBACK)
+
+
+def _positive(prometheus_name: str, selector: str) -> str:
+ return "max(max_over_time(%s%s[%s])) > 0" % (prometheus_name, selector, LOOKBACK)
+
+
+def _metric_checks(service_instance_id: str) -> Tuple[MetricCheck, ...]:
+ run = _selector(service_instance_id=service_instance_id)
+ error_probe = _selector(
+ service_instance_id=service_instance_id,
+ reader_name="load-tank-topic-error-probe",
+ retry_decision="stop",
+ status_code="SCHEME_ERROR",
+ )
+ return (
+ MetricCheck(
+ "client operation duration",
+ "ydb_client_operation_duration_seconds_count",
+ _positive("ydb_client_operation_duration_seconds_count", run),
+ ),
+ MetricCheck(
+ "failed client operations",
+ "ydb_client_operation_failed_total",
+ _positive("ydb_client_operation_failed_total", run),
+ ),
+ MetricCheck("query session count", "ydb_query_session_count", _present("ydb_query_session_count", run)),
+ MetricCheck(
+ "closed query sessions",
+ "ydb_query_session_closed_total",
+ _positive("ydb_query_session_closed_total", run),
+ ),
+ MetricCheck(
+ "query session create time",
+ "ydb_query_session_create_time_seconds_count",
+ _positive("ydb_query_session_create_time_seconds_count", run),
+ ),
+ MetricCheck(
+ "pending query session requests",
+ "ydb_query_session_pending_requests",
+ _positive("ydb_query_session_pending_requests", run),
+ ),
+ MetricCheck(
+ "query session acquire timeouts",
+ "ydb_query_session_timeouts_total",
+ _positive("ydb_query_session_timeouts_total", run),
+ ),
+ MetricCheck("query session maximum", "ydb_query_session_max", _positive("ydb_query_session_max", run)),
+ MetricCheck("query session minimum", "ydb_query_session_min", _present("ydb_query_session_min", run)),
+ MetricCheck(
+ "retry attempts above one",
+ "ydb_client_retry_attempts_count",
+ "max(max_over_time(ydb_client_retry_attempts_sum%s[%s]) "
+ "- max_over_time(ydb_client_retry_attempts_count%s[%s])) > 0" % (run, LOOKBACK, run, LOOKBACK),
+ ),
+ MetricCheck(
+ "retry duration",
+ "ydb_client_retry_duration_seconds_count",
+ _positive("ydb_client_retry_duration_seconds_count", run),
+ ),
+ MetricCheck(
+ "topic messages received",
+ "ydb_topic_reader_received_messages_total",
+ _positive("ydb_topic_reader_received_messages_total", run),
+ ),
+ MetricCheck(
+ "topic bytes received",
+ "ydb_topic_reader_received_bytes_total",
+ _positive("ydb_topic_reader_received_bytes_total", run),
+ ),
+ MetricCheck(
+ "topic messages delivered",
+ "ydb_topic_reader_delivered_messages_total",
+ _positive("ydb_topic_reader_delivered_messages_total", run),
+ ),
+ MetricCheck(
+ "topic message decompression duration",
+ "ydb_topic_reader_compression_duration_seconds_count",
+ _positive("ydb_topic_reader_compression_duration_seconds_count", run),
+ ),
+ MetricCheck(
+ "topic message processing duration",
+ "ydb_topic_reader_message_processing_duration_seconds_count",
+ _positive("ydb_topic_reader_message_processing_duration_seconds_count", run),
+ ),
+ MetricCheck(
+ "topic commit ranges queued",
+ "ydb_topic_reader_commit_queued_total",
+ _positive("ydb_topic_reader_commit_queued_total", run),
+ ),
+ MetricCheck(
+ "topic commit ranges acknowledged",
+ "ydb_topic_reader_commit_acknowledged_total",
+ _positive("ydb_topic_reader_commit_acknowledged_total", run),
+ ),
+ MetricCheck(
+ "topic commit offset lag",
+ "ydb_topic_reader_commit_offset_lag_max",
+ _present("ydb_topic_reader_commit_offset_lag_max", run),
+ ),
+ MetricCheck(
+ "topic reader session errors",
+ "ydb_topic_reader_session_errors_total",
+ _positive("ydb_topic_reader_session_errors_total", error_probe),
+ requires_error_probe=True,
+ ),
+ MetricCheck(
+ "topic reader credit balance",
+ "ydb_topic_reader_credit_balance_bytes",
+ _positive("ydb_topic_reader_credit_balance_bytes", run),
+ ),
+ MetricCheck(
+ "topic reader local buffer",
+ "ydb_topic_reader_local_buffer_messages",
+ _positive("ydb_topic_reader_local_buffer_messages", run),
+ ),
+ MetricCheck(
+ "oldest buffered topic message age",
+ "ydb_topic_reader_local_buffer_message_age_max_seconds",
+ _positive("ydb_topic_reader_local_buffer_message_age_max_seconds", run),
+ ),
+ MetricCheck(
+ "topic partition sessions",
+ "ydb_topic_reader_partition_session_count",
+ _positive("ydb_topic_reader_partition_session_count", run),
+ ),
+ )
+
+
+def _prometheus_query_has_result(prometheus_url: str, promql: str, timeout: float = 5) -> bool:
+ query = urllib.parse.urlencode({"query": promql})
+ url = "%s/api/v1/query?%s" % (prometheus_url.rstrip("/"), query)
+ with urllib.request.urlopen(url, timeout=timeout) as response:
+ payload = json.load(response)
+ return payload.get("status") == "success" and bool(payload.get("data", {}).get("result"))
+
+
+def _prometheus_has_series(prometheus_url: str, prometheus_name: str) -> bool:
+ return _prometheus_query_has_result(prometheus_url, "count(%s)" % prometheus_name)
+
+
+def wait_for_metrics(
+ prometheus_url: str,
+ checks: Iterable[MetricCheck],
+ timeout: float,
+ poll_interval: float,
+) -> Dict[str, MetricCheck]:
+ missing = {check.prometheus_name: check for check in checks}
+ deadline = time.monotonic() + timeout
+
+ while missing and time.monotonic() < deadline:
+ for prometheus_name in tuple(missing):
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+ try:
+ check = missing[prometheus_name]
+ if _prometheus_query_has_result(
+ prometheus_url,
+ check.promql or "count(%s)" % prometheus_name,
+ timeout=min(5, remaining),
+ ):
+ check = missing.pop(prometheus_name)
+ print("[ok] %s: %s" % (check.description, check.prometheus_name))
+ except (OSError, ValueError, json.JSONDecodeError):
+ pass
+
+ if missing:
+ remaining = deadline - time.monotonic()
+ if remaining > 0:
+ time.sleep(min(poll_interval, remaining))
+
+ return missing
+
+
+def _read_run_id_file(path: str, timeout: float, poll_interval: float) -> str:
+ deadline = time.monotonic() + timeout
+ while True:
+ try:
+ run_id = Path(path).read_text(encoding="utf-8").strip()
+ if run_id:
+ return run_id
+ except OSError:
+ pass
+
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise RuntimeError("Run id was not published to %s" % path)
+ time.sleep(min(poll_interval, remaining))
+
+
+def _parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--prometheus-url",
+ default=os.environ.get("PROMETHEUS_URL", "http://localhost:9090"),
+ )
+ parser.add_argument("--timeout", type=float, default=180)
+ parser.add_argument("--poll-interval", type=float, default=2)
+ parser.add_argument(
+ "--service-instance-id",
+ default=os.environ.get("METRICS_SERVICE_INSTANCE_ID", ""),
+ help="service.instance.id of the load generator run to verify",
+ )
+ parser.add_argument(
+ "--run-id-file",
+ default=os.environ.get("METRICS_RUN_ID_FILE", ""),
+ help="file from which to read the current load generator run id",
+ )
+ parser.add_argument(
+ "--require-error-probe",
+ action="store_true",
+ default=os.environ.get("REQUIRE_TOPIC_ERROR_PROBE", "").lower() in {"1", "true", "yes"},
+ help="also require ydb.topic.reader.session.errors",
+ )
+ return parser.parse_args()
+
+
+def main() -> int:
+ args = _parse_args()
+ started_at = time.monotonic()
+ if not args.service_instance_id and not args.run_id_file:
+ print("Specify --service-instance-id or --run-id-file to select the load generator run.")
+ return 2
+
+ try:
+ service_instance_id = args.service_instance_id or _read_run_id_file(
+ args.run_id_file,
+ args.timeout,
+ args.poll_interval,
+ )
+ except RuntimeError as error:
+ print("Cannot select the load generator run: %s" % error)
+ return 2
+
+ checks = tuple(
+ check
+ for check in _metric_checks(service_instance_id)
+ if args.require_error_probe or not check.requires_error_probe
+ )
+ remaining_timeout = max(0, args.timeout - (time.monotonic() - started_at))
+ print("Checking metrics for service.instance.id=%s" % service_instance_id)
+ missing = wait_for_metrics(
+ args.prometheus_url,
+ checks,
+ timeout=remaining_timeout,
+ poll_interval=args.poll_interval,
+ )
+ if not missing:
+ print("All %s expected SDK metrics are present in Prometheus." % len(checks))
+ return 0
+
+ print("Missing SDK metrics after %ss:" % args.timeout)
+ for check in missing.values():
+ print("[missing] %s: %s" % (check.description, check.prometheus_name))
+ return 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/examples/topic/reader_async_example.py b/examples/topic/reader_async_example.py
index 71e77455..464293c1 100644
--- a/examples/topic/reader_async_example.py
+++ b/examples/topic/reader_async_example.py
@@ -9,7 +9,11 @@ async def connect():
connection_string="grpc://localhost:2135?database=/local",
credentials=ydb.credentials.AnonymousCredentials(),
)
- reader = db.topic_client.reader("/local/topic", consumer="consumer")
+ reader = db.topic_client.reader(
+ "/local/topic",
+ consumer="consumer",
+ reader_name="payments-worker",
+ )
return reader
diff --git a/examples/topic/reader_example.py b/examples/topic/reader_example.py
index ab8157a9..53caab4b 100644
--- a/examples/topic/reader_example.py
+++ b/examples/topic/reader_example.py
@@ -8,7 +8,11 @@ def connect():
connection_string="grpc://localhost:2135?database=/local",
credentials=ydb.credentials.AnonymousCredentials(),
)
- reader = db.topic_client.reader("/local/topic", consumer="consumer")
+ reader = db.topic_client.reader(
+ "/local/topic",
+ consumer="consumer",
+ reader_name="payments-worker",
+ )
return reader
diff --git a/setup.py b/setup.py
index b6d81051..0b7e8473 100644
--- a/setup.py
+++ b/setup.py
@@ -38,6 +38,6 @@
install_requires=requirements, # requirements.txt
extras_require={
"yc": ["yandexcloud", ],
- "opentelemetry": ["opentelemetry-api>=1.0.0"],
+ "opentelemetry": ["opentelemetry-api>=1.12.0"],
}
)
diff --git a/tests/observability/test_metrics.py b/tests/observability/test_metrics.py
index 94c8ead6..4ab8d9fa 100644
--- a/tests/observability/test_metrics.py
+++ b/tests/observability/test_metrics.py
@@ -1,7 +1,9 @@
import asyncio
import inspect
+import time
from contextlib import contextmanager
-from unittest.mock import AsyncMock, MagicMock
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, call
import pytest
from opentelemetry.metrics import Meter
@@ -79,6 +81,71 @@ def _assert_closed_metric(reader, pool_name, reason):
)
+def _topic_reader_batch(message_count, topic, partition_session_id=1, reader_stream_id=1):
+ return SimpleNamespace(
+ messages=[object()] * message_count,
+ _partition_session=SimpleNamespace(
+ topic_path=topic,
+ id=partition_session_id,
+ reader_stream_id=reader_stream_id,
+ ),
+ )
+
+
+def _topic_reader_partition_session(topic, partition_session_id, reader_stream_id, committed_offset=0):
+ return SimpleNamespace(
+ topic_path=topic,
+ id=partition_session_id,
+ reader_stream_id=reader_stream_id,
+ committed_offset=committed_offset,
+ )
+
+
+def _topic_read_response(partition_messages, bytes_size=30):
+ import datetime
+
+ from ydb._grpc.grpcwrapper.ydb_topic import Codec, StreamReadMessage
+
+ now = datetime.datetime(2023, 2, 3, 14, 15)
+ return StreamReadMessage.ReadResponse(
+ partition_data=[
+ StreamReadMessage.ReadResponse.PartitionData(
+ partition_session_id=partition_session_id,
+ batches=[
+ StreamReadMessage.ReadResponse.Batch(
+ message_data=[
+ StreamReadMessage.ReadResponse.MessageData(
+ offset=offset,
+ seq_no=offset + 1,
+ created_at=now,
+ data=("message-%d" % offset).encode(),
+ metadata_items={},
+ uncompresed_size=10,
+ message_group_id="group",
+ )
+ for offset in range(message_count)
+ ],
+ producer_id="producer",
+ write_session_meta={},
+ codec=Codec.CODEC_RAW,
+ written_at=now,
+ )
+ ],
+ )
+ for partition_session_id, message_count in partition_messages
+ ],
+ bytes_size=bytes_size,
+ )
+
+
+async def _decode_pending_topic_batches(stream):
+ while not stream._batches_to_decode.empty():
+ batch = stream._batches_to_decode.get_nowait()
+ await stream._decode_batch_inplace(batch)
+ stream._add_batch_to_queue(batch)
+ stream._state_changed.set()
+
+
def _histogram_boundaries_advisory_supported():
return "explicit_bucket_boundaries_advisory" in inspect.signature(Meter.create_histogram).parameters
@@ -86,7 +153,7 @@ def _histogram_boundaries_advisory_supported():
def test_metrics_build_info_version(metrics_setup):
from ydb.observability.metrics import _metrics_build_info_tokens
- assert _metrics_build_info_tokens() == ["ydb-sdk-metrics/0.2.0"]
+ assert _metrics_build_info_tokens() == ["ydb-sdk-metrics/0.25.0"]
def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch):
@@ -103,10 +170,36 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch):
QUERY_SESSION_TIMEOUTS,
RETRY_ATTEMPTS,
RETRY_DURATION,
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_COMMIT_QUEUED,
+ TOPIC_READER_COMPRESSION_DURATION,
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_DELIVERED_MESSAGES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ TOPIC_READER_SESSION_ERRORS,
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ TOPIC_WRITER_BUFFER_USED_BYTES,
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_WRITER_SENDING_BYTES,
+ TOPIC_WRITER_SENDING_MESSAGES,
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ TOPIC_WRITER_SESSION_ERRORS,
+ TOPIC_WRITER_WRITTEN_MESSAGES,
ATTEMPT_BUCKETS,
DURATION_BUCKETS_SECONDS,
RETRY_DURATION_BUCKETS_SECONDS,
SessionMetrics,
+ TopicReaderMetrics,
+ TopicWriterMetrics,
create_metrics_operation,
record_query_session_count,
record_query_session_create_time,
@@ -116,7 +209,10 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch):
record_retry_metrics,
)
- monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(side_effect=[1.0, 1.25]))
+ monkeypatch.setattr(
+ "ydb.observability.metrics.time.monotonic",
+ MagicMock(return_value=10.0),
+ )
with pytest.raises(issues.Unavailable):
with create_metrics_operation("ExecuteQuery"):
@@ -133,6 +229,40 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch):
record_query_session_pending_requests(1, "main")
record_query_session_timeout("main")
record_retry_metrics(0.75, 3)
+ reader_metrics = TopicReaderMetrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="reader-test",
+ )
+ reader_metrics.record_received_messages([_topic_reader_batch(2, "/Root/events")])
+ reader_metrics.record_received_bytes(128)
+ reader_metrics.record_delivered_messages(2, "/Root/events")
+ reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events")
+ reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events")
+ reader_metrics.record_commit_queued(2, "/Root/events")
+ reader_metrics.record_commit_acknowledged(2, "/Root/events")
+ reader_metrics.record_session_error(issues.Unavailable("retry"), True)
+ reader_metrics.reset_credit_balance_bytes(1024)
+ partition_session = _topic_reader_partition_session("/Root/events", 1, 1)
+ reader_metrics.add_partition_session("/Root/events", partition_session)
+ reader_metrics.record_commit_offset_queued(partition_session, 2)
+ writer_metrics = TopicWriterMetrics(
+ object(),
+ topic="/Root/events",
+ writer_name="writer-test",
+ buffer_limit_bytes=1024,
+ )
+ writer_metrics.record_sending_messages(2)
+ writer_metrics.record_sending_bytes(128)
+ writer_metrics.record_written_messages(2)
+ writer_metrics.record_session_error(issues.Unavailable("retry"), True)
+ writer_metrics.set_buffer_used_bytes(128)
+ writer_metrics.record_buffer_wait_duration(writer_metrics.capture_buffer_wait_started_at())
+ writer_metrics.record_compression_duration(writer_metrics.capture_compression_started_at(), "gzip")
+ accepted_at = writer_metrics.capture_message_accepted_at()
+ writer_metrics.record_message_processing_duration(accepted_at, "gzip")
+ writer_metrics.record_message_ack_duration(writer_metrics.capture_message_first_sent_at())
+ writer_metrics.add_pending_messages(accepted_at, 2)
metrics = _metrics_by_name(metrics_setup)
@@ -148,6 +278,30 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch):
QUERY_SESSION_TIMEOUTS,
RETRY_ATTEMPTS,
RETRY_DURATION,
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_COMMIT_QUEUED,
+ TOPIC_READER_COMPRESSION_DURATION,
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_DELIVERED_MESSAGES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ TOPIC_READER_SESSION_ERRORS,
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ TOPIC_WRITER_BUFFER_USED_BYTES,
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_WRITER_SENDING_BYTES,
+ TOPIC_WRITER_SENDING_MESSAGES,
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ TOPIC_WRITER_SESSION_ERRORS,
+ TOPIC_WRITER_WRITTEN_MESSAGES,
}
assert metrics[CLIENT_OPERATION_DURATION].unit == "s"
assert metrics[CLIENT_OPERATION_FAILED].unit == "{command}"
@@ -160,6 +314,41 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch):
assert metrics[QUERY_SESSION_TIMEOUTS].unit == "{connection}"
assert metrics[RETRY_DURATION].unit == "s"
assert metrics[RETRY_ATTEMPTS].unit == "{attempt}"
+ assert metrics[TOPIC_READER_COMMIT_ACKNOWLEDGED].unit == "{message}"
+ assert metrics[TOPIC_READER_COMMIT_OFFSET_LAG_MAX].unit == "{message}"
+ assert metrics[TOPIC_READER_COMMIT_QUEUED].unit == "{message}"
+ assert metrics[TOPIC_READER_COMPRESSION_DURATION].unit == "s"
+ assert metrics[TOPIC_READER_CREDIT_BALANCE_BYTES].unit == "By"
+ assert metrics[TOPIC_READER_DELIVERED_MESSAGES].unit == "{message}"
+ assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX].unit == "s"
+ assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGES].unit == "{message}"
+ assert metrics[TOPIC_READER_MESSAGE_PROCESSING_DURATION].unit == "s"
+ assert metrics[TOPIC_READER_PARTITION_SESSION_COUNT].unit == "{session}"
+ assert metrics[TOPIC_READER_RECEIVED_MESSAGES].unit == "{message}"
+ assert metrics[TOPIC_READER_RECEIVED_BYTES].unit == "By"
+ assert metrics[TOPIC_READER_SESSION_ERRORS].unit == "{error}"
+ assert metrics[TOPIC_WRITER_BUFFER_LIMIT_BYTES].unit == "By"
+ assert metrics[TOPIC_WRITER_BUFFER_USED_BYTES].unit == "By"
+ assert metrics[TOPIC_WRITER_BUFFER_WAIT_DURATION].unit == "s"
+ assert metrics[TOPIC_WRITER_COMPRESSION_DURATION].unit == "s"
+ assert metrics[TOPIC_WRITER_MESSAGE_ACK_DURATION].unit == "s"
+ assert metrics[TOPIC_WRITER_MESSAGE_PROCESSING_DURATION].unit == "s"
+ assert metrics[TOPIC_WRITER_SENDING_BYTES].unit == "By"
+ assert metrics[TOPIC_WRITER_SENDING_MESSAGES].unit == "{message}"
+ assert metrics[TOPIC_WRITER_SENDING_OLDEST_AGE].unit == "s"
+ assert metrics[TOPIC_WRITER_SESSION_ERRORS].unit == "{error}"
+ assert metrics[TOPIC_WRITER_WRITTEN_MESSAGES].unit == "{message}"
+ assert metrics[TOPIC_READER_CREDIT_BALANCE_BYTES].data.__class__.__name__ == "Gauge"
+ assert metrics[TOPIC_READER_COMMIT_OFFSET_LAG_MAX].data.__class__.__name__ == "Gauge"
+ assert metrics[TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX].data.__class__.__name__ == "Gauge"
+ assert metrics[TOPIC_READER_PARTITION_SESSION_COUNT].data.__class__.__name__ == "Sum"
+ assert metrics[TOPIC_WRITER_BUFFER_LIMIT_BYTES].data.__class__.__name__ == "Gauge"
+ assert metrics[TOPIC_WRITER_BUFFER_USED_BYTES].data.__class__.__name__ == "Gauge"
+ assert metrics[TOPIC_WRITER_BUFFER_WAIT_DURATION].data.__class__.__name__ == "Histogram"
+ assert metrics[TOPIC_WRITER_COMPRESSION_DURATION].data.__class__.__name__ == "Histogram"
+ assert metrics[TOPIC_WRITER_MESSAGE_ACK_DURATION].data.__class__.__name__ == "Histogram"
+ assert metrics[TOPIC_WRITER_MESSAGE_PROCESSING_DURATION].data.__class__.__name__ == "Histogram"
+ assert metrics[TOPIC_WRITER_SENDING_OLDEST_AGE].data.__class__.__name__ == "Gauge"
if _histogram_boundaries_advisory_supported():
assert (
_single_point_from_metrics(metrics, CLIENT_OPERATION_DURATION).explicit_bounds == DURATION_BUCKETS_SECONDS
@@ -169,6 +358,30 @@ def test_metrics_registry_records_all_instruments(metrics_setup, monkeypatch):
)
assert _single_point_from_metrics(metrics, RETRY_DURATION).explicit_bounds == RETRY_DURATION_BUCKETS_SECONDS
assert _single_point_from_metrics(metrics, RETRY_ATTEMPTS).explicit_bounds == ATTEMPT_BUCKETS
+ assert (
+ _single_point_from_metrics(metrics, TOPIC_READER_COMPRESSION_DURATION).explicit_bounds
+ == DURATION_BUCKETS_SECONDS
+ )
+ assert (
+ _single_point_from_metrics(metrics, TOPIC_READER_MESSAGE_PROCESSING_DURATION).explicit_bounds
+ == DURATION_BUCKETS_SECONDS
+ )
+ assert (
+ _single_point_from_metrics(metrics, TOPIC_WRITER_BUFFER_WAIT_DURATION).explicit_bounds
+ == DURATION_BUCKETS_SECONDS
+ )
+ assert (
+ _single_point_from_metrics(metrics, TOPIC_WRITER_COMPRESSION_DURATION).explicit_bounds
+ == DURATION_BUCKETS_SECONDS
+ )
+ assert (
+ _single_point_from_metrics(metrics, TOPIC_WRITER_MESSAGE_ACK_DURATION).explicit_bounds
+ == DURATION_BUCKETS_SECONDS
+ )
+ assert (
+ _single_point_from_metrics(metrics, TOPIC_WRITER_MESSAGE_PROCESSING_DURATION).explicit_bounds
+ == DURATION_BUCKETS_SECONDS
+ )
def test_metrics_registry_supports_old_histogram_api():
@@ -198,6 +411,9 @@ def create_up_down_counter(self, name, unit="", description=""):
def create_observable_up_down_counter(self, name, callbacks=None, unit="", description=""):
return FakeInstrument()
+ def create_observable_gauge(self, name, callbacks=None, unit="", description=""):
+ return FakeInstrument()
+
OtelMetricsProvider(FakeMeter())
@@ -224,11 +440,15 @@ def test_metrics_registry_is_noop_without_meter(monkeypatch):
_NOOP_PROVIDER,
_NOOP_QUERY_SESSION_POOL_METRICS,
_NOOP_SESSION_METRICS,
+ _NOOP_TOPIC_READER_METRICS,
+ _NOOP_TOPIC_WRITER_METRICS,
QuerySessionPoolMetrics,
SessionMetrics,
create_query_session_pool_metrics,
create_session_metrics,
create_metrics_operation,
+ create_topic_reader_metrics,
+ create_topic_writer_metrics,
record_query_session_create_time,
record_query_session_max,
record_query_session_pending_requests,
@@ -288,6 +508,63 @@ def test_metrics_registry_is_noop_without_meter(monkeypatch):
noop_pool_metrics.on_released(noop_session)
noop_pool_metrics.close()
+ def fail_on_iteration():
+ raise AssertionError("noop topic reader metrics iterated over batches")
+ yield
+
+ noop_reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="reader-test",
+ topics=fail_on_iteration(),
+ )
+ assert noop_reader_metrics is _NOOP_TOPIC_READER_METRICS
+
+ noop_reader_metrics.record_received_messages(fail_on_iteration())
+ assert noop_reader_metrics.capture_received_at() is None
+ noop_reader_metrics.record_received_bytes(10)
+ noop_reader_metrics.record_delivered_messages(1, "/Root/events")
+ noop_reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events")
+ noop_reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events")
+ noop_reader_metrics.record_commit_queued(1, "/Root/events")
+ noop_reader_metrics.record_commit_acknowledged(1, "/Root/events")
+ noop_reader_metrics.record_commit_offset_queued(object(), 1)
+ noop_reader_metrics.record_commit_offset_acknowledged(object(), 1)
+ noop_reader_metrics.record_session_error(RuntimeError("ignored"), True)
+ noop_reader_metrics.add_credit_balance_bytes(-10)
+ noop_reader_metrics.reset_credit_balance_bytes(10)
+ noop_reader_metrics.remove_local_buffer_messages(1, object())
+ noop_reader_metrics.reset_local_buffer_messages()
+ noop_reader_metrics.add_partition_session("/Root/events")
+ noop_reader_metrics.remove_partition_session("/Root/events")
+ noop_reader_metrics.reset_partition_sessions()
+ noop_reader_metrics.close()
+
+ noop_writer_metrics = create_topic_writer_metrics(
+ object(),
+ topic="/Root/events",
+ writer_name="writer-test",
+ buffer_limit_bytes=1024,
+ )
+ assert noop_writer_metrics is _NOOP_TOPIC_WRITER_METRICS
+ noop_writer_metrics.record_sending_messages(1)
+ noop_writer_metrics.record_sending_bytes(10)
+ noop_writer_metrics.record_written_messages(1)
+ noop_writer_metrics.record_session_error(RuntimeError("ignored"), True)
+ assert noop_writer_metrics.capture_buffer_wait_started_at() is None
+ noop_writer_metrics.record_buffer_wait_duration(None)
+ assert noop_writer_metrics.capture_message_first_sent_at() is None
+ noop_writer_metrics.record_message_ack_duration(None)
+ assert noop_writer_metrics.capture_compression_started_at() is None
+ noop_writer_metrics.record_compression_duration(None, "gzip")
+ assert noop_writer_metrics.capture_message_accepted_at() is None
+ noop_writer_metrics.record_message_processing_duration(None, "gzip")
+ noop_writer_metrics.add_pending_messages(None, 1)
+ noop_writer_metrics.complete_pending_message()
+ noop_writer_metrics.clear_pending_messages()
+ noop_writer_metrics.set_buffer_used_bytes(10)
+ noop_writer_metrics.close()
+
assert not monotonic.called
assert not noop_add.called
pool_metrics.close()
@@ -883,6 +1160,167 @@ def test_reconfigure_metrics_quiesces_old_gauge_callbacks(self):
first_provider.shutdown()
second_provider.shutdown()
+ def test_topic_reader_gauges_keep_state_while_metrics_are_reconfigured(self, monkeypatch):
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ create_topic_reader_metrics,
+ )
+ from ydb.opentelemetry import disable_metrics, enable_metrics
+
+ first_reader = InMemoryMetricReader()
+ first_provider = MeterProvider(metric_readers=[first_reader])
+ second_reader = InMemoryMetricReader()
+ second_provider = MeterProvider(metric_readers=[second_reader])
+
+ disable_metrics()
+ enable_metrics(first_provider)
+ reader_metrics = create_topic_reader_metrics(object(), consumer_name=None, reader_name="reconfigured")
+ first_batch = _topic_reader_batch(2, "/Root/events")
+ reader_metrics.record_received_messages(
+ [first_batch],
+ received_at=10.0,
+ )
+ reader_metrics.reset_credit_balance_bytes(100)
+ partition_session = _topic_reader_partition_session("/Root/events", 1, 1)
+ reader_metrics.add_partition_session("/Root/events", partition_session)
+ reader_metrics.record_commit_offset_queued(partition_session, 5)
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=30.0))
+ try:
+ assert _single_point(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 2
+ assert _single_point(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0
+ assert _single_point(first_reader, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1
+ assert _single_point(first_reader, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 100
+ assert _single_point(first_reader, TOPIC_READER_COMMIT_OFFSET_LAG_MAX).value == 5
+
+ disable_metrics()
+ second_batch = _topic_reader_batch(3, "/Root/events")
+ reader_metrics.record_received_messages(
+ [second_batch],
+ received_at=20.0,
+ )
+ reader_metrics.remove_local_buffer_messages(1, first_batch._partition_session)
+ reader_metrics.add_credit_balance_bytes(-20)
+ reader_metrics.record_commit_offset_acknowledged(partition_session, 2)
+ reader_metrics.add_partition_session("/Root/events")
+ enable_metrics(second_provider)
+
+ assert _points(first_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == []
+ assert _points(first_reader, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) == []
+ assert _single_point(second_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 4
+ assert _single_point(second_reader, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0
+ assert _single_point(second_reader, TOPIC_READER_PARTITION_SESSION_COUNT).value == 2
+ assert _single_point(second_reader, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 80
+ assert _single_point(second_reader, TOPIC_READER_COMMIT_OFFSET_LAG_MAX).value == 3
+ finally:
+ reader_metrics.close()
+ disable_metrics()
+ first_provider.shutdown()
+ second_provider.shutdown()
+
+ def test_topic_writer_histograms_finish_on_their_original_provider(self, monkeypatch):
+ from ydb.observability import disable_metrics, enable_metrics
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ create_topic_writer_metrics,
+ )
+
+ class RecordingProvider:
+ def __init__(self):
+ self.records = []
+
+ def record(self, name, value, attributes=None):
+ self.records.append((name, value, attributes))
+
+ def add(self, name, value, attributes=None):
+ pass
+
+ def observe_gauge(self, name, callback):
+ pass
+
+ first_provider = RecordingProvider()
+ second_provider = RecordingProvider()
+ clock = MagicMock(side_effect=[10.0, 20.0, 30.0, 40.0, 10.25, 20.5, 30.75, 41.0])
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", clock)
+
+ disable_metrics()
+ enable_metrics(first_provider)
+ writer_metrics = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="reconfigured")
+ try:
+ wait_started_at = writer_metrics.capture_buffer_wait_started_at()
+ accepted_at = writer_metrics.capture_message_accepted_at()
+ compression_started_at = writer_metrics.capture_compression_started_at()
+ first_sent_at = writer_metrics.capture_message_first_sent_at()
+
+ enable_metrics(second_provider)
+ writer_metrics.record_buffer_wait_duration(wait_started_at)
+ writer_metrics.record_message_processing_duration(accepted_at, "gzip")
+ writer_metrics.record_compression_duration(compression_started_at, "gzip")
+ writer_metrics.record_message_ack_duration(first_sent_at)
+
+ assert [(name, value) for name, value, _ in first_provider.records] == [
+ (TOPIC_WRITER_BUFFER_WAIT_DURATION, 0.25),
+ (TOPIC_WRITER_MESSAGE_PROCESSING_DURATION, 0.5),
+ (TOPIC_WRITER_COMPRESSION_DURATION, 0.75),
+ (TOPIC_WRITER_MESSAGE_ACK_DURATION, 1.0),
+ ]
+ assert second_provider.records == []
+ finally:
+ writer_metrics.close()
+ disable_metrics()
+
+ def test_topic_writer_oldest_age_keeps_state_while_metrics_are_reconfigured(self, monkeypatch):
+ from ydb.observability import disable_metrics, enable_metrics
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ create_topic_writer_metrics,
+ )
+
+ class RecordingProvider:
+ def __init__(self):
+ self.gauges = {}
+
+ def record(self, name, value, attributes=None):
+ pass
+
+ def add(self, name, value, attributes=None):
+ pass
+
+ def observe_gauge(self, name, callback):
+ self.gauges[name] = callback
+
+ first_provider = RecordingProvider()
+ second_provider = RecordingProvider()
+
+ disable_metrics()
+ enable_metrics(first_provider)
+ writer_metrics = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="reconfigured")
+ try:
+ disable_metrics()
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=10.0))
+ accepted_at = writer_metrics.capture_message_accepted_at()
+ writer_metrics.add_pending_messages(accepted_at, 1)
+
+ enable_metrics(second_provider)
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=15.0))
+
+ assert first_provider.gauges[TOPIC_WRITER_SENDING_OLDEST_AGE]() == ()
+ observations = list(second_provider.gauges[TOPIC_WRITER_SENDING_OLDEST_AGE]())
+ assert [(value, attrs["writer.name"]) for value, attrs in observations] == [(5.0, "reconfigured")]
+
+ writer_metrics.complete_pending_message()
+ observations = list(second_provider.gauges[TOPIC_WRITER_SENDING_OLDEST_AGE]())
+ assert [(value, attrs["writer.name"]) for value, attrs in observations] == [(0.0, "reconfigured")]
+ finally:
+ writer_metrics.close()
+ disable_metrics()
+
def test_gauge_callback_drops_snapshot_if_provider_changes_during_collection(self):
from ydb.observability import disable_metrics, enable_metrics
from ydb.observability import metrics
@@ -1038,107 +1476,1620 @@ def fake_import(name, *args, **kwargs):
otel.disable_metrics()
-class TestEndpointHelpers:
- @pytest.mark.parametrize(
- "endpoint,expected",
- [
- ("grpc://localhost:2136", ("localhost", 2136)),
- ("grpcs://localhost:2136", ("localhost", 2136)),
- ("localhost:2135", ("localhost", 2135)),
- ("localhost", ("localhost", 0)),
- ("[::1]:2136", ("[::1]", 2136)),
- ("grpc://[2001:db8::1]:2136", ("[2001:db8::1]", 2136)),
- ("host:badport", ("host", 0)),
- (None, ("", 0)),
- ],
+def test_topic_writer_message_metrics_attributes(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ TOPIC_WRITER_BUFFER_USED_BYTES,
+ TOPIC_WRITER_SENDING_BYTES,
+ TOPIC_WRITER_SENDING_MESSAGES,
+ TOPIC_WRITER_WRITTEN_MESSAGES,
+ create_topic_writer_metrics,
)
- def test_split_endpoint(self, endpoint, expected):
- from ydb.observability._endpoint import split_endpoint
- assert split_endpoint(endpoint) == expected
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
- def test_build_ydb_metrics_attrs(self):
- from tests.observability.conftest import FakeDriverConfig
- from ydb.observability.metrics import _build_ydb_metrics_attrs
+ writer_metrics = create_topic_writer_metrics(
+ FakeDriver(),
+ topic="/Root/events",
+ writer_name="payments-writer",
+ buffer_limit_bytes=1024,
+ )
+ writer_metrics.record_sending_messages(2)
+ writer_metrics.record_sending_bytes(128)
+ writer_metrics.record_sending_messages(3)
+ writer_metrics.record_sending_bytes(256)
+ writer_metrics.record_written_messages(2)
+ writer_metrics.record_written_messages(1)
+ writer_metrics.set_buffer_used_bytes(128)
+
+ buffer_limit = _single_point(metrics_setup, TOPIC_WRITER_BUFFER_LIMIT_BYTES)
+ buffer_used = _single_point(metrics_setup, TOPIC_WRITER_BUFFER_USED_BYTES)
+ messages = _single_point(metrics_setup, TOPIC_WRITER_SENDING_MESSAGES)
+ sending_bytes = _single_point(metrics_setup, TOPIC_WRITER_SENDING_BYTES)
+ written = _single_point(metrics_setup, TOPIC_WRITER_WRITTEN_MESSAGES)
+ expected_attributes = {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "topic": "/Root/events",
+ "writer.name": "payments-writer",
+ }
+ assert (messages.value, messages.attributes) == (
+ 5,
+ expected_attributes,
+ )
+ assert (sending_bytes.value, sending_bytes.attributes) == (384, expected_attributes)
+ assert (written.value, written.attributes) == (3, expected_attributes)
+ assert (buffer_limit.value, buffer_limit.attributes) == (1024, expected_attributes)
+ assert (buffer_used.value, buffer_used.attributes) == (128, expected_attributes)
+ writer_metrics.close()
- attrs = _build_ydb_metrics_attrs(FakeDriverConfig(endpoint="grpc://node:2136", database="/db"))
- assert attrs == {"database": "/db", "endpoint": "node:2136"}
+def test_topic_writer_buffer_wait_duration_records_only_completed_wait(metrics_setup, monkeypatch):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ create_topic_writer_metrics,
+ )
-class TestTracingTelemetryFacade:
- def test_telemetry_operation_forwards_to_span_and_metrics(self, metrics_setup, otel_setup):
- from tests.observability.conftest import FakeDriverConfig
- from ydb.observability.tracing import create_ydb_span
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
- telemetry = create_ydb_span("ydb.ExecuteQuery", FakeDriverConfig())
- telemetry.set_attribute("database", "/db")
- telemetry.set_error(ValueError("fail"))
- telemetry.end()
+ values = iter([10.0, 10.25])
+ monotonic = MagicMock(side_effect=lambda: next(values, 10.25))
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", monotonic)
+ writer_metrics = create_topic_writer_metrics(
+ FakeDriver(),
+ topic="/Root/events",
+ writer_name="payments-writer",
+ )
+ try:
+ started_at = writer_metrics.capture_buffer_wait_started_at()
+ assert started_at[1] == 10.0
+ writer_metrics.record_buffer_wait_duration(None)
+ assert monotonic.call_count == 1
+
+ writer_metrics.record_buffer_wait_duration(started_at)
+ assert monotonic.call_count == 2
+ point = _single_point(metrics_setup, TOPIC_WRITER_BUFFER_WAIT_DURATION)
+ assert (point.count, point.sum, point.attributes) == (
+ 1,
+ 0.25,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "topic": "/Root/events",
+ "writer.name": "payments-writer",
+ },
+ )
+ writer_metrics.record_buffer_wait_duration(None)
+ finally:
+ writer_metrics.close()
- spans = otel_setup.get_finished_spans()
- assert len(spans) == 1
- assert spans[0].status.status_code.name == "ERROR"
- def test_telemetry_context_end_on_exit_false(self, metrics_setup, otel_setup):
- from tests.observability.conftest import FakeDriverConfig
- from ydb.observability.metrics import CLIENT_OPERATION_DURATION
- from ydb.observability.tracing import create_ydb_span
+def test_topic_writer_message_processing_duration_records_outcomes(metrics_setup, monkeypatch):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ create_topic_writer_metrics,
+ )
- telemetry = create_ydb_span("ydb.ExecuteQuery", FakeDriverConfig())
- with telemetry.attach_context(end_on_exit=False):
- pass
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
- assert CLIENT_OPERATION_DURATION not in _metrics_by_name(metrics_setup)
- assert len(otel_setup.get_finished_spans()) == 0
+ values = iter([10.0, 10.25, 11.0, 11.5])
+ monotonic = MagicMock(side_effect=lambda: next(values, 11.5))
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", monotonic)
+ writer_metrics = create_topic_writer_metrics(
+ FakeDriver(),
+ topic="/Root/events",
+ writer_name="payments-writer",
+ )
+ try:
+ accepted_at = writer_metrics.capture_message_accepted_at()
+ writer_metrics.record_message_processing_duration(accepted_at, "gzip")
+ failed_at = writer_metrics.capture_message_accepted_at()
+ writer_metrics.record_message_processing_duration(failed_at, "gzip", ValueError("encode failed"))
+ assert monotonic.call_count == 4
+
+ points = _points(metrics_setup, TOPIC_WRITER_MESSAGE_PROCESSING_DURATION)
+ assert len(points) == 2
+ by_outcome = {point.attributes["outcome"]: point for point in points}
+ common_attributes = {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "topic": "/Root/events",
+ "writer.name": "payments-writer",
+ "codec": "gzip",
+ }
+ assert (by_outcome["ok"].count, by_outcome["ok"].sum, by_outcome["ok"].attributes) == (
+ 1,
+ 0.25,
+ {**common_attributes, "outcome": "ok"},
+ )
+ assert (
+ by_outcome["error"].count,
+ by_outcome["error"].sum,
+ by_outcome["error"].attributes,
+ ) == (
+ 1,
+ 0.5,
+ {**common_attributes, "outcome": "error", "error.type": "ValueError"},
+ )
+ finally:
+ writer_metrics.close()
- telemetry.end()
- assert len(otel_setup.get_finished_spans()) == 1
- assert CLIENT_OPERATION_DURATION in _metrics_by_name(metrics_setup)
- def test_set_peer_attributes_skips_none(self):
- from ydb.observability.tracing import set_peer_attributes
+def test_topic_writer_message_ack_duration_records_server_ack(metrics_setup, monkeypatch):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ create_topic_writer_metrics,
+ )
- span = MagicMock()
- set_peer_attributes(span, None)
- span.set_attribute.assert_not_called()
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
- def test_set_peer_attributes_sets_network_fields(self):
- from ydb.observability.tracing import set_peer_attributes
+ values = iter([10.0, 10.5])
+ monotonic = MagicMock(side_effect=lambda: next(values, 10.5))
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", monotonic)
+ writer_metrics = create_topic_writer_metrics(
+ FakeDriver(),
+ topic="/Root/events",
+ writer_name="payments-writer",
+ )
+ try:
+ first_sent_at = writer_metrics.capture_message_first_sent_at()
+ writer_metrics.record_message_ack_duration(first_sent_at)
+ writer_metrics.record_message_ack_duration(None)
+ assert monotonic.call_count == 2
+
+ point = _single_point(metrics_setup, TOPIC_WRITER_MESSAGE_ACK_DURATION)
+ assert (point.count, point.sum, point.attributes) == (
+ 1,
+ 0.5,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "topic": "/Root/events",
+ "writer.name": "payments-writer",
+ },
+ )
+ finally:
+ writer_metrics.close()
- span = MagicMock()
- set_peer_attributes(span, ("node.example.net", 2136, "dc-a"))
- span.set_attribute.assert_any_call("network.peer.address", "node.example.net")
- span.set_attribute.assert_any_call("network.peer.port", 2136)
- span.set_attribute.assert_any_call("ydb.node.dc", "dc-a")
+def test_topic_writer_compression_duration_records_outcomes(metrics_setup, monkeypatch):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ create_topic_writer_metrics,
+ )
- def test_span_finish_callback_records_error(self):
- from ydb.observability.tracing import span_finish_callback
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
- span = MagicMock()
- finish = span_finish_callback(span)
- finish(RuntimeError("boom"))
+ values = iter([20.0, 20.125, 21.0, 21.25])
+ monotonic = MagicMock(side_effect=lambda: next(values, 21.25))
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", monotonic)
+ writer_metrics = create_topic_writer_metrics(
+ FakeDriver(),
+ topic="/Root/events",
+ writer_name="payments-writer",
+ )
+ try:
+ started_at = writer_metrics.capture_compression_started_at()
+ writer_metrics.record_compression_duration(started_at, "gzip")
+ failed_at = writer_metrics.capture_compression_started_at()
+ writer_metrics.record_compression_duration(failed_at, "gzip", ValueError("encode failed"))
+ assert monotonic.call_count == 4
+
+ points = _points(metrics_setup, TOPIC_WRITER_COMPRESSION_DURATION)
+ assert len(points) == 2
+ by_outcome = {point.attributes["outcome"]: point for point in points}
+ common_attributes = {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "topic": "/Root/events",
+ "writer.name": "payments-writer",
+ "codec": "gzip",
+ }
+ assert (by_outcome["ok"].count, by_outcome["ok"].sum, by_outcome["ok"].attributes) == (
+ 1,
+ 0.125,
+ {**common_attributes, "outcome": "ok"},
+ )
+ assert (
+ by_outcome["error"].count,
+ by_outcome["error"].sum,
+ by_outcome["error"].attributes,
+ ) == (
+ 1,
+ 0.25,
+ {**common_attributes, "outcome": "error", "error.type": "ValueError"},
+ )
+ finally:
+ writer_metrics.close()
- span.set_error.assert_called_once()
- span.end.assert_called_once()
- def test_tracing_plugin_sets_transport_error(self, otel_setup):
- from opentelemetry import trace
- from ydb import issues
- from ydb.opentelemetry.plugin import _set_error_on_span
+def test_topic_writer_buffer_gauges_aggregate_and_unregister(metrics_setup):
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ TOPIC_WRITER_BUFFER_USED_BYTES,
+ create_topic_writer_metrics,
+ )
- error = issues.Error("lost")
- error.status = issues.StatusCode.CONNECTION_LOST
+ first = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="shared", buffer_limit_bytes=100)
+ second = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="shared", buffer_limit_bytes=200)
+ try:
+ assert _single_point(metrics_setup, TOPIC_WRITER_BUFFER_LIMIT_BYTES).value == 300
+ assert _single_point(metrics_setup, TOPIC_WRITER_BUFFER_USED_BYTES).value == 0
- raw_span = trace.get_tracer("test").start_span("x")
- _set_error_on_span(raw_span, error)
- raw_span.end()
+ first.set_buffer_used_bytes(74)
+ second.set_buffer_used_bytes(94)
+ assert _single_point(metrics_setup, TOPIC_WRITER_BUFFER_USED_BYTES).value == 168
- finished = otel_setup.get_finished_spans()[0]
- assert finished.attributes["error.type"] == "transport_error"
+ first.close()
+ assert _single_point(metrics_setup, TOPIC_WRITER_BUFFER_LIMIT_BYTES).value == 200
+ assert _single_point(metrics_setup, TOPIC_WRITER_BUFFER_USED_BYTES).value == 94
+ finally:
+ first.close()
+ second.close()
- def test_tracing_plugin_sets_generic_error_type(self, otel_setup):
- from opentelemetry import trace
+ assert _points(metrics_setup, TOPIC_WRITER_BUFFER_LIMIT_BYTES) == []
+ assert _points(metrics_setup, TOPIC_WRITER_BUFFER_USED_BYTES) == []
+
+
+def test_topic_writer_oldest_age_uses_oldest_pending_message_and_unregisters(metrics_setup, monkeypatch):
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ create_topic_writer_metrics,
+ )
+
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=20.0))
+ first = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="shared")
+ second = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="shared")
+ try:
+ clock = MagicMock(side_effect=[10.0, 15.0])
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", clock)
+ first.add_pending_messages(first.capture_message_accepted_at(), 2)
+ second.add_pending_messages(second.capture_message_accepted_at(), 1)
+ clock.side_effect = None
+ clock.return_value = 20.0
+ assert _single_point(metrics_setup, TOPIC_WRITER_SENDING_OLDEST_AGE).value == 10.0
+
+ first.complete_pending_message()
+ assert _single_point(metrics_setup, TOPIC_WRITER_SENDING_OLDEST_AGE).value == 10.0
+
+ first.complete_pending_message()
+ assert _single_point(metrics_setup, TOPIC_WRITER_SENDING_OLDEST_AGE).value == 5.0
+
+ second.clear_pending_messages()
+ assert _single_point(metrics_setup, TOPIC_WRITER_SENDING_OLDEST_AGE).value == 0.0
+ finally:
+ first.close()
+ second.close()
+
+ assert _points(metrics_setup, TOPIC_WRITER_SENDING_OLDEST_AGE) == []
+
+
+def test_topic_writer_buffer_limit_omits_unbounded_aggregate(metrics_setup):
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ create_topic_writer_metrics,
+ )
+
+ bounded = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="shared", buffer_limit_bytes=100)
+ unbounded = create_topic_writer_metrics(object(), topic="/Root/events", writer_name="shared")
+ try:
+ assert _points(metrics_setup, TOPIC_WRITER_BUFFER_LIMIT_BYTES) == []
+
+ unbounded.close()
+ assert _single_point(metrics_setup, TOPIC_WRITER_BUFFER_LIMIT_BYTES).value == 100
+ finally:
+ bounded.close()
+ unbounded.close()
+
+ assert _points(metrics_setup, TOPIC_WRITER_BUFFER_LIMIT_BYTES) == []
+
+
+@pytest.mark.parametrize("writer_name", [None, ""])
+def test_topic_writer_metrics_require_explicit_writer_name(metrics_setup, writer_name):
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ TOPIC_WRITER_BUFFER_USED_BYTES,
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_WRITER_SENDING_BYTES,
+ TOPIC_WRITER_SENDING_MESSAGES,
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ TOPIC_WRITER_SESSION_ERRORS,
+ TOPIC_WRITER_WRITTEN_MESSAGES,
+ create_topic_writer_metrics,
+ )
+
+ writer_metrics = create_topic_writer_metrics(
+ object(),
+ topic="/Root/events",
+ writer_name=writer_name,
+ buffer_limit_bytes=1024,
+ )
+ writer_metrics.record_sending_messages(1)
+ writer_metrics.record_sending_bytes(10)
+ writer_metrics.record_written_messages(1)
+ writer_metrics.record_session_error(issues.BadRequest("stop"), False)
+ assert writer_metrics.capture_buffer_wait_started_at() is None
+ writer_metrics.record_buffer_wait_duration(None)
+ assert writer_metrics.capture_message_first_sent_at() is None
+ writer_metrics.record_message_ack_duration(None)
+ assert writer_metrics.capture_compression_started_at() is None
+ writer_metrics.record_compression_duration(None, "gzip")
+ assert writer_metrics.capture_message_accepted_at() is None
+ writer_metrics.record_message_processing_duration(None, "gzip")
+ writer_metrics.add_pending_messages(None, 1)
+ writer_metrics.complete_pending_message()
+ writer_metrics.clear_pending_messages()
+ writer_metrics.set_buffer_used_bytes(10)
+ writer_metrics.close()
+
+ assert TOPIC_WRITER_BUFFER_LIMIT_BYTES not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_BUFFER_USED_BYTES not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_BUFFER_WAIT_DURATION not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_COMPRESSION_DURATION not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_MESSAGE_ACK_DURATION not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_MESSAGE_PROCESSING_DURATION not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_SENDING_MESSAGES not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_SENDING_OLDEST_AGE not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_SENDING_BYTES not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_SESSION_ERRORS not in _metrics_by_name(metrics_setup)
+ assert TOPIC_WRITER_WRITTEN_MESSAGES not in _metrics_by_name(metrics_setup)
+
+
+def test_topic_writer_session_errors_follow_retry_decision(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_SESSION_ERRORS,
+ create_topic_writer_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ writer_metrics = create_topic_writer_metrics(
+ FakeDriver(),
+ topic="/Root/events",
+ writer_name="payments-writer",
+ )
+ writer_metrics.record_session_error(issues.ConnectionLost("retry"), True)
+ writer_metrics.record_session_error(issues.ConnectionLost("retry"), True)
+ writer_metrics.record_session_error(issues.BadRequest("stop"), False)
+
+ points = _points(metrics_setup, TOPIC_WRITER_SESSION_ERRORS)
+ common_attributes = {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "topic": "/Root/events",
+ "writer.name": "payments-writer",
+ }
+ assert {point.attributes["retry_decision"]: (point.value, point.attributes) for point in points} == {
+ "retry": (
+ 2,
+ dict(
+ common_attributes,
+ retry_decision="retry",
+ status_code="CONNECTION_LOST",
+ **{"error.type": "transport_error"},
+ ),
+ ),
+ "stop": (
+ 1,
+ dict(
+ common_attributes,
+ retry_decision="stop",
+ status_code="BAD_REQUEST",
+ **{"error.type": "ydb_error"},
+ ),
+ ),
+ }
+
+
+def test_topic_reader_received_metrics_attributes(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ create_topic_reader_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ FakeDriver(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ reader_metrics.record_received_messages([_topic_reader_batch(2, "/Root/events")])
+ reader_metrics.record_received_bytes(128)
+ reader_metrics.record_received_messages([_topic_reader_batch(3, "/Root/events")])
+ reader_metrics.record_received_bytes(256)
+
+ expected_reader_attributes = {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ }
+ expected_message_attributes = dict(expected_reader_attributes, topic="/Root/events")
+ messages = _single_point(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES)
+ received_bytes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES)
+ assert (messages.value, messages.attributes) == (5, expected_message_attributes)
+ assert (received_bytes.value, received_bytes.attributes) == (384, expected_reader_attributes)
+
+
+def test_topic_reader_received_messages_are_split_by_topic(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ create_topic_reader_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ FakeDriver(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ reader_metrics.record_received_messages(
+ [
+ _topic_reader_batch(2, "/Root/orders"),
+ _topic_reader_batch(3, "/Root/refunds"),
+ _topic_reader_batch(1, "/Root/orders"),
+ ]
+ )
+ reader_metrics.record_received_bytes(128)
+
+ message_points = _points(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES)
+ assert sorted((point.attributes["topic"], point.value) for point in message_points) == [
+ ("/Root/orders", 3),
+ ("/Root/refunds", 3),
+ ]
+
+ received_bytes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES)
+ assert received_bytes.value == 128
+ assert "topic" not in received_bytes.attributes
+
+
+def test_topic_reader_received_messages_are_aggregated_before_provider_calls(monkeypatch):
+ from ydb.observability import metrics
+
+ provider = MagicMock()
+ monkeypatch.setattr(metrics, "_provider", provider)
+ reader_metrics = metrics.TopicReaderMetrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ try:
+ reader_metrics.record_received_messages(
+ [
+ _topic_reader_batch(2, "/Root/orders"),
+ _topic_reader_batch(3, "/Root/refunds"),
+ _topic_reader_batch(1, "/Root/orders"),
+ ]
+ )
+ finally:
+ reader_metrics.close()
+
+ base_attributes = {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ }
+ assert provider.add.call_args_list == [
+ call(
+ metrics.TOPIC_READER_RECEIVED_MESSAGES,
+ 3,
+ dict(base_attributes, topic="/Root/orders"),
+ ),
+ call(
+ metrics.TOPIC_READER_RECEIVED_MESSAGES,
+ 3,
+ dict(base_attributes, topic="/Root/refunds"),
+ ),
+ ]
+
+
+def test_topic_reader_delivered_messages_are_split_by_topic(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_READER_DELIVERED_MESSAGES,
+ create_topic_reader_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ FakeDriver(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ reader_metrics.record_delivered_messages(2, "/Root/orders")
+ reader_metrics.record_delivered_messages(3, "/Root/refunds")
+ reader_metrics.record_delivered_messages(1, "/Root/orders")
+ reader_metrics.record_delivered_messages(0, "/Root/ignored")
+
+ points = _points(metrics_setup, TOPIC_READER_DELIVERED_MESSAGES)
+ values = {point.attributes["topic"]: (point.value, point.attributes) for point in points}
+ assert values == {
+ "/Root/orders": (
+ 3,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/orders",
+ },
+ ),
+ "/Root/refunds": (
+ 3,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/refunds",
+ },
+ ),
+ }
+
+
+def test_topic_reader_processing_histograms_record_success_and_error_per_message(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMPRESSION_DURATION,
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION,
+ create_topic_reader_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ FakeDriver(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ error = ValueError("cannot decode")
+ reader_metrics.record_compression_duration(0.01, "gzip", "/Root/orders")
+ reader_metrics.record_message_processing_duration(0.04, "gzip", "/Root/orders")
+ reader_metrics.record_compression_duration(0.02, "gzip", "/Root/orders", error)
+ reader_metrics.record_message_processing_duration(0.06, "gzip", "/Root/orders", error)
+
+ expected_base = {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/orders",
+ "codec": "gzip",
+ }
+ compression = {
+ point.attributes["outcome"]: point for point in _points(metrics_setup, TOPIC_READER_COMPRESSION_DURATION)
+ }
+ processing = {
+ point.attributes["outcome"]: point for point in _points(metrics_setup, TOPIC_READER_MESSAGE_PROCESSING_DURATION)
+ }
+
+ assert (compression["ok"].count, compression["ok"].sum, compression["ok"].attributes) == (
+ 1,
+ pytest.approx(0.01),
+ dict(expected_base, outcome="ok"),
+ )
+ assert (processing["ok"].count, processing["ok"].sum, processing["ok"].attributes) == (
+ 1,
+ pytest.approx(0.04),
+ dict(expected_base, outcome="ok"),
+ )
+ assert (compression["error"].count, compression["error"].sum, compression["error"].attributes) == (
+ 1,
+ pytest.approx(0.02),
+ dict(expected_base, outcome="error", **{"error.type": "ValueError"}),
+ )
+ assert (processing["error"].count, processing["error"].sum, processing["error"].attributes) == (
+ 1,
+ pytest.approx(0.06),
+ dict(expected_base, outcome="error", **{"error.type": "ValueError"}),
+ )
+
+
+def test_topic_reader_commit_queued_messages_are_split_by_topic(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMMIT_QUEUED,
+ create_topic_reader_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ FakeDriver(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ reader_metrics.record_commit_queued(2, "/Root/orders")
+ reader_metrics.record_commit_queued(3, "/Root/refunds")
+ reader_metrics.record_commit_queued(1, "/Root/orders")
+ reader_metrics.record_commit_queued(0, "/Root/ignored")
+
+ points = _points(metrics_setup, TOPIC_READER_COMMIT_QUEUED)
+ values = {point.attributes["topic"]: (point.value, point.attributes) for point in points}
+ assert values == {
+ "/Root/orders": (
+ 3,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/orders",
+ },
+ ),
+ "/Root/refunds": (
+ 3,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/refunds",
+ },
+ ),
+ }
+
+
+def test_topic_reader_commit_acknowledged_messages_are_split_by_topic(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ create_topic_reader_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ FakeDriver(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ reader_metrics.record_commit_acknowledged(2, "/Root/orders")
+ reader_metrics.record_commit_acknowledged(3, "/Root/refunds")
+ reader_metrics.record_commit_acknowledged(1, "/Root/orders")
+ reader_metrics.record_commit_acknowledged(0, "/Root/ignored")
+
+ points = _points(metrics_setup, TOPIC_READER_COMMIT_ACKNOWLEDGED)
+ values = {point.attributes["topic"]: (point.value, point.attributes) for point in points}
+ assert values == {
+ "/Root/orders": (
+ 3,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/orders",
+ },
+ ),
+ "/Root/refunds": (
+ 3,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/refunds",
+ },
+ ),
+ }
+
+
+def test_topic_reader_commit_offset_lag_tracks_maximum_across_partitions(metrics_setup):
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ create_topic_reader_metrics,
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ first = _topic_reader_partition_session("/Root/orders", 1, 10, committed_offset=10)
+ second = _topic_reader_partition_session("/Root/orders", 2, 10, committed_offset=100)
+ other = _topic_reader_partition_session("/Root/refunds", 3, 10, committed_offset=20)
+ try:
+ reader_metrics.add_partition_session(first.topic_path, first)
+ reader_metrics.add_partition_session(second.topic_path, second)
+ reader_metrics.add_partition_session(other.topic_path, other)
+
+ assert {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX)
+ } == {"/Root/orders": 0, "/Root/refunds": 0}
+
+ reader_metrics.record_commit_offset_queued(first, 15)
+ reader_metrics.record_commit_offset_queued(second, 108)
+ reader_metrics.record_commit_offset_queued(other, 23)
+ assert {
+ point.attributes["topic"]: (point.value, point.attributes)
+ for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX)
+ } == {
+ "/Root/orders": (
+ 8,
+ {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/orders",
+ },
+ ),
+ "/Root/refunds": (
+ 3,
+ {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/refunds",
+ },
+ ),
+ }
+
+ reader_metrics.record_commit_offset_acknowledged(second, 108)
+ assert {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX)
+ } == {"/Root/orders": 5, "/Root/refunds": 3}
+
+ reader_metrics.record_commit_offset_acknowledged(first, 9)
+ assert {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX)
+ } == {"/Root/orders": 5, "/Root/refunds": 3}
+
+ reader_metrics.record_commit_offset_acknowledged(first, 15)
+ reader_metrics.record_commit_offset_acknowledged(other, 23)
+ assert {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX)
+ } == {"/Root/orders": 0, "/Root/refunds": 0}
+
+ reader_metrics.remove_partition_session(first.topic_path, first)
+ reader_metrics.remove_partition_session(second.topic_path, second)
+ assert {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX)
+ } == {"/Root/refunds": 0}
+
+ reader_metrics.reset_partition_sessions()
+ assert _points(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX) == []
+ finally:
+ reader_metrics.close()
+
+
+def test_topic_reader_session_errors_are_split_by_retry_decision(metrics_setup):
+ from ydb.observability.metrics import (
+ TOPIC_READER_SESSION_ERRORS,
+ create_topic_reader_metrics,
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ reader_metrics.record_session_error(issues.Unavailable("retry"), True)
+ reader_metrics.record_session_error(issues.Unavailable("retry"), True)
+ reader_metrics.record_session_error(issues.BadRequest("stop"), False)
+
+ points = _points(metrics_setup, TOPIC_READER_SESSION_ERRORS)
+ assert {point.attributes["retry_decision"]: (point.value, point.attributes) for point in points} == {
+ "retry": (
+ 2,
+ {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "retry_decision": "retry",
+ "status_code": "UNAVAILABLE",
+ "error.type": "ydb_error",
+ },
+ ),
+ "stop": (
+ 1,
+ {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "retry_decision": "stop",
+ "status_code": "BAD_REQUEST",
+ "error.type": "ydb_error",
+ },
+ ),
+ }
+
+
+@pytest.mark.parametrize(
+ ("error", "status_code", "error_type"),
+ [
+ (issues.ConnectionLost("lost"), "CONNECTION_LOST", "transport_error"),
+ (issues.ConnectionError("connection"), "ConnectionError", "transport_error"),
+ (issues.BadRequest("bad request"), "BAD_REQUEST", "ydb_error"),
+ (RuntimeError("unexpected"), "RuntimeError", "RuntimeError"),
+ ],
+)
+def test_topic_reader_session_errors_classify_failure(metrics_setup, error, status_code, error_type):
+ from ydb.observability.metrics import (
+ TOPIC_READER_SESSION_ERRORS,
+ create_topic_reader_metrics,
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ reader_metrics.record_session_error(error, True)
+
+ point = _single_point(metrics_setup, TOPIC_READER_SESSION_ERRORS)
+ assert point.attributes["status_code"] == status_code
+ assert point.attributes["error.type"] == error_type
+
+
+def test_topic_reader_credit_balance_tracks_stream_credit(metrics_setup):
+ from ydb.observability.metrics import (
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ create_topic_reader_metrics,
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ try:
+ assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 0
+
+ reader_metrics.reset_credit_balance_bytes(100)
+ assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 100
+
+ reader_metrics.add_credit_balance_bytes(-12)
+ assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 88
+
+ reader_metrics.add_credit_balance_bytes(4)
+ assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 92
+
+ reader_metrics.reset_credit_balance_bytes()
+ assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 0
+ finally:
+ reader_metrics.close()
+
+ assert _points(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES) == []
+
+
+def test_topic_reader_local_buffer_messages_tracks_current_reader_buffer(metrics_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import (
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ create_topic_reader_metrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ FakeDriver(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ try:
+ orders_batch = _topic_reader_batch(2, "/Root/orders", partition_session_id=1)
+ refunds_batch = _topic_reader_batch(3, "/Root/refunds", partition_session_id=2)
+ reader_metrics.record_received_messages([orders_batch, refunds_batch])
+
+ point = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES)
+ assert (point.value, point.attributes) == (
+ 5,
+ {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ },
+ )
+ assert "topic" not in point.attributes
+
+ reader_metrics.remove_local_buffer_messages(2, orders_batch._partition_session)
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 3
+
+ reader_metrics.reset_local_buffer_messages()
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0
+ finally:
+ reader_metrics.close()
+
+ assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == []
+
+
+def test_topic_reader_local_buffer_age_tracks_oldest_message_per_partition(metrics_setup, monkeypatch):
+ from ydb.observability import metrics
+
+ reader_metrics = metrics.create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ )
+ first_partition_early = _topic_reader_batch(
+ 1,
+ "/Root/events",
+ partition_session_id=1,
+ )
+ second_partition = _topic_reader_batch(
+ 1,
+ "/Root/events",
+ partition_session_id=2,
+ )
+ first_partition_late = _topic_reader_batch(
+ 1,
+ "/Root/events",
+ partition_session_id=1,
+ )
+
+ try:
+ assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 0
+
+ reader_metrics.record_received_messages([first_partition_early], received_at=10.0)
+ reader_metrics.record_received_messages([second_partition], received_at=20.0)
+ reader_metrics.record_received_messages([first_partition_late], received_at=30.0)
+ monkeypatch.setattr(metrics.time, "monotonic", MagicMock(return_value=40.0))
+
+ age = _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX)
+ assert (age.value, age.attributes) == (
+ 30.0,
+ {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ },
+ )
+
+ reader_metrics.remove_local_buffer_messages(1, second_partition._partition_session)
+ assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 30.0
+
+ reader_metrics.remove_local_buffer_messages(1, first_partition_early._partition_session)
+ assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 10.0
+
+ reader_metrics.remove_local_buffer_messages(1, first_partition_late._partition_session)
+ assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 0
+ assert _single_point(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0
+ finally:
+ reader_metrics.close()
+
+ assert _points(metrics_setup, metrics.TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX) == []
+
+
+def test_topic_reader_partition_session_count_is_split_by_topic(metrics_setup):
+ from ydb.observability.metrics import (
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ create_topic_reader_metrics,
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name="payments-worker",
+ topics=("/Root/orders", "/Root/refunds"),
+ )
+ try:
+ assert {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT)
+ } == {
+ "/Root/orders": 0,
+ "/Root/refunds": 0,
+ }
+
+ reader_metrics.add_partition_session("/Root/orders")
+ reader_metrics.add_partition_session("/Root/orders")
+ reader_metrics.add_partition_session("/Root/refunds")
+
+ points = _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT)
+ assert {point.attributes["topic"]: (point.value, point.attributes) for point in points} == {
+ "/Root/orders": (
+ 2,
+ {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/orders",
+ },
+ ),
+ "/Root/refunds": (
+ 1,
+ {
+ "endpoint": "",
+ "database": "",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ "topic": "/Root/refunds",
+ },
+ ),
+ }
+
+ reader_metrics.remove_partition_session("/Root/orders")
+ reader_metrics.remove_partition_session("/Root/refunds")
+ points = _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT)
+ assert {point.attributes["topic"]: point.value for point in points} == {
+ "/Root/orders": 1,
+ "/Root/refunds": 0,
+ }
+
+ reader_metrics.reset_partition_sessions()
+ assert {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT)
+ } == {
+ "/Root/orders": 0,
+ "/Root/refunds": 0,
+ }
+ finally:
+ reader_metrics.close()
+
+ assert _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) == []
+
+
+@pytest.mark.asyncio
+async def test_topic_reader_metrics_follow_reader_lifecycle_across_topics(metrics_setup):
+ from unittest import mock
+
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb._topic_reader.datatypes import PartitionSession
+ from ydb._topic_reader.topic_reader import PublicReaderSettings
+ from ydb._topic_reader.topic_reader_asyncio import PublicAsyncIOReader, ReaderStream
+ from ydb.observability.metrics import (
+ TOPIC_READER_DELIVERED_MESSAGES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig(
+ endpoint="grpc://localhost:2136",
+ database="/Root",
+ )
+
+ settings = PublicReaderSettings(
+ consumer="analytics",
+ topic="/Root/orders",
+ reader_name="payments-worker",
+ )
+ stream_ready = asyncio.Event()
+ stream_holder = []
+
+ async def create_stream(reader_reconnector_id, driver, stream_settings, *, reader_name, metrics):
+ stream = ReaderStream(
+ reader_reconnector_id,
+ stream_settings,
+ reader_name=reader_name,
+ metrics=metrics,
+ )
+ for session_id, topic in [(1, "/Root/orders"), (2, "/Root/refunds")]:
+ stream._partition_sessions[session_id] = PartitionSession(
+ id=session_id,
+ state=PartitionSession.State.Active,
+ topic_path=topic,
+ partition_id=session_id,
+ committed_offset=0,
+ reader_reconnector_id=reader_reconnector_id,
+ reader_stream_id=stream._id,
+ )
+ stream_holder.append(stream)
+ stream_ready.set()
+ return stream
+
+ reader_attributes = {
+ "endpoint": "localhost:2136",
+ "database": "/Root",
+ "consumer": "analytics",
+ "reader.name": "payments-worker",
+ }
+ with mock.patch.object(ReaderStream, "create", create_stream):
+ reader = PublicAsyncIOReader(FakeDriver(), settings)
+ try:
+ await asyncio.wait_for(stream_ready.wait(), timeout=1)
+ stream = stream_holder[0]
+ assert (
+ _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value,
+ _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).attributes,
+ ) == (0, reader_attributes)
+
+ stream._on_read_response(_topic_read_response([(1, 2), (2, 1)]))
+ await _decode_pending_topic_batches(stream)
+
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 3
+ assert len(_points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES)) == 1
+ assert _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES).value == 30
+
+ assert await reader.receive_message() is not None
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 2
+ assert len((await reader.receive_batch(max_messages=1)).messages) == 1
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 1
+ assert len((await reader.receive_batch()).messages) == 1
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0
+
+ received = {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES)
+ }
+ delivered = {
+ point.attributes["topic"]: point.value
+ for point in _points(metrics_setup, TOPIC_READER_DELIVERED_MESSAGES)
+ }
+ assert received == {"/Root/orders": 2, "/Root/refunds": 1}
+ assert delivered == received
+ finally:
+ await reader.close(flush=False)
+
+ assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == []
+
+
+@pytest.mark.asyncio
+async def test_topic_reader_gauges_reset_across_stream_reconnect(metrics_setup, monkeypatch):
+ from unittest import mock
+
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb._topic_reader.datatypes import PartitionSession
+ from ydb._topic_reader.topic_reader import PublicReaderSettings
+ from ydb._topic_reader.topic_reader_asyncio import PublicAsyncIOReader, ReaderStream
+ from ydb.observability.metrics import (
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ TopicReaderMetrics,
+ )
+
+ class FakeDriver:
+ _driver_config = FakeDriverConfig()
+
+ settings = PublicReaderSettings(
+ consumer="analytics",
+ topic="/Root/orders",
+ reader_name="reconnecting-reader",
+ )
+ first_stream_ready = asyncio.Event()
+ second_stream_create_started = asyncio.Event()
+ allow_second_stream = asyncio.Event()
+ second_stream_ready = asyncio.Event()
+ streams = []
+ monkeypatch.setattr(
+ TopicReaderMetrics,
+ "capture_received_at",
+ lambda self: time.monotonic() - 5.0,
+ )
+
+ async def create_stream(reader_reconnector_id, driver, stream_settings, *, reader_name, metrics):
+ stream = ReaderStream(
+ reader_reconnector_id,
+ stream_settings,
+ reader_name=reader_name,
+ metrics=metrics,
+ )
+ stream._partition_sessions[1] = PartitionSession(
+ id=1,
+ state=PartitionSession.State.Active,
+ topic_path="/Root/orders",
+ partition_id=1,
+ committed_offset=0,
+ reader_reconnector_id=reader_reconnector_id,
+ reader_stream_id=stream._id,
+ )
+ streams.append(stream)
+ if len(streams) == 1:
+ metrics.reset_credit_balance_bytes(stream_settings.buffer_size_bytes)
+ metrics.add_partition_session("/Root/orders")
+ stream._on_read_response(_topic_read_response([(1, 2)], bytes_size=20))
+ await _decode_pending_topic_batches(stream)
+ first_stream_ready.set()
+ else:
+ second_stream_create_started.set()
+ await allow_second_stream.wait()
+ metrics.reset_credit_balance_bytes(stream_settings.buffer_size_bytes)
+ metrics.add_partition_session("/Root/orders")
+ stream._on_read_response(_topic_read_response([(1, 1)], bytes_size=10))
+ await _decode_pending_topic_batches(stream)
+ second_stream_ready.set()
+ return stream
+
+ with mock.patch.object(ReaderStream, "create", create_stream):
+ reader = PublicAsyncIOReader(FakeDriver(), settings)
+ try:
+ await asyncio.wait_for(first_stream_ready.wait(), timeout=1)
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 2
+ first_age = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value
+ assert 5.0 <= first_age < 10.0
+ assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1
+ assert (
+ _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == settings.buffer_size_bytes - 20
+ )
+
+ streams[0]._set_first_error(issues.Unavailable("reconnect"))
+ await asyncio.wait_for(second_stream_create_started.wait(), timeout=2)
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 0
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 0
+ assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 0
+ assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 0
+
+ allow_second_stream.set()
+ await asyncio.wait_for(second_stream_ready.wait(), timeout=1)
+
+ async def wait_until_second_stream_is_installed():
+ while reader._reconnector._stream_reader is not streams[1]:
+ await asyncio.sleep(0)
+
+ await asyncio.wait_for(wait_until_second_stream_is_installed(), timeout=1)
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 1
+ second_age = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value
+ assert 5.0 <= second_age < 10.0
+ assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 1
+ assert (
+ _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == settings.buffer_size_bytes - 10
+ )
+ finally:
+ allow_second_stream.set()
+ await reader.close(flush=False)
+
+ assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES) == []
+ assert _points(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX) == []
+ assert _points(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT) == []
+ assert _points(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES) == []
+
+
+def test_topic_reader_gauges_aggregate_duplicate_reader_attributes(metrics_setup, monkeypatch):
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ create_topic_reader_metrics,
+ )
+
+ first = create_topic_reader_metrics(object(), consumer_name=None, reader_name="shared")
+ second = create_topic_reader_metrics(object(), consumer_name=None, reader_name="shared")
+ first_partition = _topic_reader_partition_session("/Root/events", 1, 1)
+ second_partition = _topic_reader_partition_session("/Root/events", 2, 1)
+ third_partition = _topic_reader_partition_session("/Root/events", 3, 2)
+ try:
+ first.record_received_messages([_topic_reader_batch(2, "/Root/events")], received_at=10.0)
+ second.record_received_messages([_topic_reader_batch(3, "/Root/events")], received_at=20.0)
+ first.reset_credit_balance_bytes(100)
+ second.reset_credit_balance_bytes(50)
+ first.add_partition_session("/Root/events", first_partition)
+ first.add_partition_session("/Root/events", second_partition)
+ second.add_partition_session("/Root/events", third_partition)
+ first.record_commit_offset_queued(first_partition, 5)
+ first.record_commit_offset_queued(second_partition, 2)
+ second.record_commit_offset_queued(third_partition, 8)
+ monkeypatch.setattr("ydb.observability.metrics.time.monotonic", MagicMock(return_value=30.0))
+
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).value == 5
+ assert _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX).value == 20.0
+ assert _single_point(metrics_setup, TOPIC_READER_PARTITION_SESSION_COUNT).value == 3
+ assert _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).value == 150
+ assert _single_point(metrics_setup, TOPIC_READER_COMMIT_OFFSET_LAG_MAX).value == 8
+ finally:
+ first.close()
+ second.close()
+
+
+def test_topic_reader_metrics_omit_consumer(metrics_setup):
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_COMMIT_QUEUED,
+ TOPIC_READER_COMPRESSION_DURATION,
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_DELIVERED_MESSAGES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ TOPIC_READER_SESSION_ERRORS,
+ create_topic_reader_metrics,
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name=None,
+ reader_name="reader-42",
+ )
+ try:
+ reader_metrics.record_received_messages([_topic_reader_batch(1, "/Root/events")])
+ reader_metrics.record_received_bytes(10)
+ reader_metrics.record_delivered_messages(1, "/Root/events")
+ reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events")
+ reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events")
+ reader_metrics.record_commit_queued(1, "/Root/events")
+ reader_metrics.record_commit_acknowledged(1, "/Root/events")
+ reader_metrics.record_session_error(issues.BadRequest("stop"), False)
+ reader_metrics.reset_credit_balance_bytes(100)
+ partition_session = _topic_reader_partition_session("/Root/events", 1, 1)
+ reader_metrics.add_partition_session("/Root/events", partition_session)
+ reader_metrics.record_commit_offset_queued(partition_session, 1)
+
+ reader_attributes = {
+ "endpoint": "",
+ "database": "",
+ "reader.name": "reader-42",
+ }
+ message_attributes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_MESSAGES).attributes
+ bytes_attributes = _single_point(metrics_setup, TOPIC_READER_RECEIVED_BYTES).attributes
+ delivered_attributes = _single_point(metrics_setup, TOPIC_READER_DELIVERED_MESSAGES).attributes
+ compression_attributes = _single_point(metrics_setup, TOPIC_READER_COMPRESSION_DURATION).attributes
+ processing_attributes = _single_point(metrics_setup, TOPIC_READER_MESSAGE_PROCESSING_DURATION).attributes
+ commit_queued_attributes = _single_point(metrics_setup, TOPIC_READER_COMMIT_QUEUED).attributes
+ commit_acknowledged_attributes = _single_point(
+ metrics_setup,
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ ).attributes
+ commit_offset_lag_attributes = _single_point(
+ metrics_setup,
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ ).attributes
+ session_error_attributes = _single_point(metrics_setup, TOPIC_READER_SESSION_ERRORS).attributes
+ credit_attributes = _single_point(metrics_setup, TOPIC_READER_CREDIT_BALANCE_BYTES).attributes
+ buffer_attributes = _single_point(metrics_setup, TOPIC_READER_LOCAL_BUFFER_MESSAGES).attributes
+ buffer_age_attributes = _single_point(
+ metrics_setup,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ ).attributes
+ partition_session_attributes = _single_point(
+ metrics_setup,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ ).attributes
+ assert message_attributes == dict(reader_attributes, topic="/Root/events")
+ assert bytes_attributes == reader_attributes
+ assert delivered_attributes == dict(reader_attributes, topic="/Root/events")
+ duration_attributes = dict(reader_attributes, topic="/Root/events", codec="gzip", outcome="ok")
+ assert compression_attributes == duration_attributes
+ assert processing_attributes == duration_attributes
+ assert commit_queued_attributes == dict(reader_attributes, topic="/Root/events")
+ assert commit_acknowledged_attributes == dict(reader_attributes, topic="/Root/events")
+ assert commit_offset_lag_attributes == dict(reader_attributes, topic="/Root/events")
+ assert session_error_attributes == dict(
+ reader_attributes,
+ retry_decision="stop",
+ status_code="BAD_REQUEST",
+ **{"error.type": "ydb_error"},
+ )
+ assert credit_attributes == reader_attributes
+ assert buffer_attributes == reader_attributes
+ assert buffer_age_attributes == reader_attributes
+ assert partition_session_attributes == dict(reader_attributes, topic="/Root/events")
+ finally:
+ reader_metrics.close()
+
+
+@pytest.mark.parametrize("reader_name", [None, ""])
+def test_topic_reader_metrics_require_explicit_reader_name(metrics_setup, reader_name):
+ from ydb.observability.metrics import (
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_COMMIT_QUEUED,
+ TOPIC_READER_COMPRESSION_DURATION,
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_DELIVERED_MESSAGES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ TOPIC_READER_SESSION_ERRORS,
+ create_topic_reader_metrics,
+ )
+
+ reader_metrics = create_topic_reader_metrics(
+ object(),
+ consumer_name="analytics",
+ reader_name=reader_name,
+ )
+ reader_metrics.record_received_messages([_topic_reader_batch(1, "/Root/events")])
+ reader_metrics.record_received_bytes(10)
+ reader_metrics.record_delivered_messages(1, "/Root/events")
+ reader_metrics.record_compression_duration(0.01, "gzip", "/Root/events")
+ reader_metrics.record_message_processing_duration(0.02, "gzip", "/Root/events")
+ reader_metrics.record_commit_queued(1, "/Root/events")
+ reader_metrics.record_commit_acknowledged(1, "/Root/events")
+ reader_metrics.record_session_error(issues.Unavailable("retry"), True)
+ reader_metrics.reset_credit_balance_bytes(100)
+ reader_metrics.add_partition_session("/Root/events")
+
+ metrics = _metrics_by_name(metrics_setup)
+ assert TOPIC_READER_RECEIVED_MESSAGES not in metrics
+ assert TOPIC_READER_RECEIVED_BYTES not in metrics
+ assert TOPIC_READER_COMMIT_ACKNOWLEDGED not in metrics
+ assert TOPIC_READER_COMMIT_OFFSET_LAG_MAX not in metrics
+ assert TOPIC_READER_COMMIT_QUEUED not in metrics
+ assert TOPIC_READER_COMPRESSION_DURATION not in metrics
+ assert TOPIC_READER_SESSION_ERRORS not in metrics
+ assert TOPIC_READER_CREDIT_BALANCE_BYTES not in metrics
+ assert TOPIC_READER_DELIVERED_MESSAGES not in metrics
+ assert TOPIC_READER_LOCAL_BUFFER_MESSAGES not in metrics
+ assert TOPIC_READER_MESSAGE_PROCESSING_DURATION not in metrics
+ assert TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX not in metrics
+ assert TOPIC_READER_PARTITION_SESSION_COUNT not in metrics
+
+
+class TestEndpointHelpers:
+ @pytest.mark.parametrize(
+ "endpoint,expected",
+ [
+ ("grpc://localhost:2136", ("localhost", 2136)),
+ ("grpcs://localhost:2136", ("localhost", 2136)),
+ ("localhost:2135", ("localhost", 2135)),
+ ("localhost", ("localhost", 0)),
+ ("[::1]:2136", ("[::1]", 2136)),
+ ("grpc://[2001:db8::1]:2136", ("[2001:db8::1]", 2136)),
+ ("host:badport", ("host", 0)),
+ (None, ("", 0)),
+ ],
+ )
+ def test_split_endpoint(self, endpoint, expected):
+ from ydb.observability._endpoint import split_endpoint
+
+ assert split_endpoint(endpoint) == expected
+
+ def test_build_ydb_metrics_attrs(self):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import _build_ydb_metrics_attrs
+
+ attrs = _build_ydb_metrics_attrs(FakeDriverConfig(endpoint="grpc://node:2136", database="/db"))
+ assert attrs == {"database": "/db", "endpoint": "node:2136"}
+
+
+class TestTracingTelemetryFacade:
+ def test_telemetry_operation_forwards_to_span_and_metrics(self, metrics_setup, otel_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.tracing import create_ydb_span
+
+ telemetry = create_ydb_span("ydb.ExecuteQuery", FakeDriverConfig())
+ telemetry.set_attribute("database", "/db")
+ telemetry.set_error(ValueError("fail"))
+ telemetry.end()
+
+ spans = otel_setup.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code.name == "ERROR"
+
+ def test_telemetry_context_end_on_exit_false(self, metrics_setup, otel_setup):
+ from tests.observability.conftest import FakeDriverConfig
+ from ydb.observability.metrics import CLIENT_OPERATION_DURATION
+ from ydb.observability.tracing import create_ydb_span
+
+ telemetry = create_ydb_span("ydb.ExecuteQuery", FakeDriverConfig())
+ with telemetry.attach_context(end_on_exit=False):
+ pass
+
+ assert CLIENT_OPERATION_DURATION not in _metrics_by_name(metrics_setup)
+ assert len(otel_setup.get_finished_spans()) == 0
+
+ telemetry.end()
+ assert len(otel_setup.get_finished_spans()) == 1
+ assert CLIENT_OPERATION_DURATION in _metrics_by_name(metrics_setup)
+
+ def test_set_peer_attributes_skips_none(self):
+ from ydb.observability.tracing import set_peer_attributes
+
+ span = MagicMock()
+ set_peer_attributes(span, None)
+ span.set_attribute.assert_not_called()
+
+ def test_set_peer_attributes_sets_network_fields(self):
+ from ydb.observability.tracing import set_peer_attributes
+
+ span = MagicMock()
+ set_peer_attributes(span, ("node.example.net", 2136, "dc-a"))
+
+ span.set_attribute.assert_any_call("network.peer.address", "node.example.net")
+ span.set_attribute.assert_any_call("network.peer.port", 2136)
+ span.set_attribute.assert_any_call("ydb.node.dc", "dc-a")
+
+ def test_span_finish_callback_records_error(self):
+ from ydb.observability.tracing import span_finish_callback
+
+ span = MagicMock()
+ finish = span_finish_callback(span)
+ finish(RuntimeError("boom"))
+
+ span.set_error.assert_called_once()
+ span.end.assert_called_once()
+
+ def test_tracing_plugin_sets_transport_error(self, otel_setup):
+ from opentelemetry import trace
+ from ydb import issues
+ from ydb.opentelemetry.plugin import _set_error_on_span
+
+ error = issues.Error("lost")
+ error.status = issues.StatusCode.CONNECTION_LOST
+
+ raw_span = trace.get_tracer("test").start_span("x")
+ _set_error_on_span(raw_span, error)
+ raw_span.end()
+
+ finished = otel_setup.get_finished_spans()[0]
+ assert finished.attributes["error.type"] == "transport_error"
+
+ def test_tracing_plugin_sets_generic_error_type(self, otel_setup):
+ from opentelemetry import trace
from ydb.opentelemetry.plugin import _set_error_on_span
raw_span = trace.get_tracer("test").start_span("x")
@@ -1698,3 +3649,263 @@ def fake_span_ctx(**kwargs):
assert qs._session_metrics._counted
assert _single_point_for_pool(metrics_setup, QUERY_SESSION_COUNT, "async-session-pool").value == 1
qs._close_session()
+
+
+def test_opentelemetry_load_tank_topic_payload_has_exact_size():
+ from examples.opentelemetry.load_tank import _topic_payload
+
+ assert _topic_payload(42, 64).startswith(b"42:")
+ assert len(_topic_payload(42, 64)) == 64
+ assert len(_topic_payload(42, 1)) == 1
+
+
+@pytest.mark.asyncio
+async def test_opentelemetry_load_tank_retry_probe_records_multiple_attempts(metrics_setup, monkeypatch):
+ import asyncio
+ from types import SimpleNamespace
+
+ import ydb
+ from examples.opentelemetry.load_tank import _retry_probe_worker
+ from ydb.observability.metrics import RETRY_ATTEMPTS
+
+ stop = asyncio.Event()
+ retry_operation_async = ydb.retry_operation_async
+
+ async def run_once(*args, **kwargs):
+ result = await retry_operation_async(*args, **kwargs)
+ stop.set()
+ return result
+
+ monkeypatch.setattr(ydb, "retry_operation_async", run_once)
+ await _retry_probe_worker(
+ SimpleNamespace(retry_probe_interval=1.0, retry_probe_failures=2),
+ stop,
+ )
+
+ retry_attempts = _single_point(metrics_setup, RETRY_ATTEMPTS)
+ assert retry_attempts.count == 1
+ assert retry_attempts.sum == 3
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("query_error", "should_raise"),
+ [
+ (issues.SchemeError("expected query failure"), False),
+ (RuntimeError("unexpected failure"), True),
+ ],
+)
+async def test_opentelemetry_load_tank_error_worker_only_suppresses_expected_error(
+ query_error,
+ should_raise,
+):
+ from examples.opentelemetry.load_tank import _error_worker
+
+ stop = asyncio.Event()
+
+ async def execute_with_retries(_):
+ stop.set()
+ raise query_error
+
+ pool = SimpleNamespace(execute_with_retries=execute_with_retries)
+ config = SimpleNamespace(error_query="missing table", error_rps=1)
+ if should_raise:
+ with pytest.raises(RuntimeError, match="unexpected failure"):
+ await _error_worker(pool, config, stop)
+ else:
+ await _error_worker(pool, config, stop)
+
+
+def test_opentelemetry_metric_verifier_detects_prometheus_series(monkeypatch):
+ from examples.opentelemetry.verify_metrics import _prometheus_has_series
+
+ class FakeResponse:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_):
+ pass
+
+ def read(self):
+ return b'{"status":"success","data":{"result":[{"value":[1,"1"]}]}}'
+
+ urlopen = MagicMock(return_value=FakeResponse())
+ monkeypatch.setattr("urllib.request.urlopen", urlopen)
+
+ assert _prometheus_has_series("http://prometheus:9090", "ydb_query_session_count")
+ assert "query=count%28ydb_query_session_count%29" in urlopen.call_args.args[0]
+
+
+def test_opentelemetry_metric_verifier_scopes_every_check_to_current_run():
+ from examples.opentelemetry.verify_metrics import _metric_checks
+
+ checks = _metric_checks('current-run"with\\escaping')
+
+ assert all('service_instance_id="current-run\\"with\\\\escaping"' in check.promql for check in checks)
+ error_check = next(check for check in checks if check.requires_error_probe)
+ assert 'reader_name="load-tank-topic-error-probe"' in error_check.promql
+ assert 'retry_decision="stop"' in error_check.promql
+ assert 'status_code="SCHEME_ERROR"' in error_check.promql
+
+
+def test_opentelemetry_metric_verifier_respects_total_timeout(monkeypatch):
+ from examples.opentelemetry import verify_metrics
+
+ now = 100.0
+ request_timeouts = []
+
+ def monotonic():
+ return now
+
+ def query_has_result(_, __, timeout):
+ nonlocal now
+ request_timeouts.append(timeout)
+ now += timeout
+ return False
+
+ monkeypatch.setattr(verify_metrics.time, "monotonic", monotonic)
+ monkeypatch.setattr(verify_metrics, "_prometheus_query_has_result", query_has_result)
+
+ checks = (
+ verify_metrics.MetricCheck("first", "first_metric"),
+ verify_metrics.MetricCheck("second", "second_metric"),
+ )
+ missing = verify_metrics.wait_for_metrics(
+ "http://prometheus:9090",
+ checks,
+ timeout=3,
+ poll_interval=10,
+ )
+
+ assert request_timeouts == [3]
+ assert tuple(missing) == ("first_metric", "second_metric")
+
+
+def test_opentelemetry_dashboard_keeps_topic_reader_instances_separate():
+ import json
+ from pathlib import Path
+
+ dashboard = json.loads(
+ Path("examples/opentelemetry/grafana/dashboards/ydb-python-sdk-metrics.json").read_text(encoding="utf-8")
+ )
+ topic_reader_targets = [
+ target
+ for panel in dashboard["panels"]
+ for target in panel.get("targets", ())
+ if "ydb_topic_reader_" in target.get("expr", "")
+ ]
+
+ assert topic_reader_targets
+ for target in topic_reader_targets:
+ assert all(label in target["expr"] for label in ("service_instance_id", "endpoint", "database", "consumer"))
+
+
+def test_opentelemetry_load_tank_publishes_current_run_id():
+ from pathlib import Path
+ from tempfile import TemporaryDirectory
+
+ from examples.opentelemetry.load_tank import _publish_run_id
+
+ with TemporaryDirectory(dir=".") as directory:
+ run_id_file = Path(directory) / "metrics" / "run-id"
+ ready_file = Path(directory) / "ready"
+ _publish_run_id("current-run", str(run_id_file), str(ready_file))
+
+ assert run_id_file.read_text(encoding="utf-8") == "current-run\n"
+ assert ready_file.is_file()
+
+
+@pytest.mark.asyncio
+async def test_opentelemetry_load_tank_propagates_background_failure():
+ from examples.opentelemetry.load_tank import _await_with_background
+
+ async def fail():
+ raise ValueError("topic failed")
+
+ task = asyncio.create_task(fail())
+ with pytest.raises(RuntimeError, match="topic stream worker failed") as error:
+ await _await_with_background(asyncio.sleep(60), {task: "topic stream worker"})
+
+ assert isinstance(error.value.__cause__, ValueError)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("cancelled", [False, True])
+async def test_opentelemetry_load_tank_detects_stopped_background_task(cancelled):
+ from examples.opentelemetry.load_tank import _await_with_background
+
+ task = asyncio.create_task(asyncio.sleep(0 if not cancelled else 60))
+ if cancelled:
+ task.cancel()
+
+ message = "worker was cancelled unexpectedly" if cancelled else "worker stopped unexpectedly"
+ with pytest.raises(RuntimeError, match=message):
+ await _await_with_background(asyncio.sleep(60), {task: "worker"})
+
+
+@pytest.mark.asyncio
+async def test_opentelemetry_load_tank_propagates_failure_during_worker_cancellation():
+ from examples.opentelemetry.load_tank import _cancel_background_tasks
+
+ async def fail_during_cancellation():
+ try:
+ await asyncio.sleep(60)
+ except asyncio.CancelledError as error:
+ raise ValueError("cleanup failed") from error
+
+ task = asyncio.create_task(fail_during_cancellation())
+ await asyncio.sleep(0)
+ with pytest.raises(RuntimeError, match="worker failed") as error:
+ await _cancel_background_tasks({task: "worker"}, raise_failures=True)
+
+ assert isinstance(error.value.__cause__, ValueError)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("probe_error", "expected_error"),
+ [
+ (issues.SchemeError("missing topic"), None),
+ (RuntimeError("unexpected failure"), RuntimeError),
+ ],
+)
+async def test_opentelemetry_load_tank_topic_error_probe_is_strict(
+ monkeypatch,
+ probe_error,
+ expected_error,
+):
+ import ydb
+ from examples.opentelemetry.load_tank import _topic_error_probe
+
+ class Reader:
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *_):
+ pass
+
+ async def wait_message(self):
+ raise probe_error
+
+ class Driver:
+ def __init__(self, **_):
+ self.topic_client = SimpleNamespace(reader=lambda *args, **kwargs: Reader())
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *_):
+ pass
+
+ monkeypatch.setattr(ydb.aio, "Driver", Driver)
+ config = SimpleNamespace(
+ database="/local",
+ topic_error_probe_endpoint="grpc://localhost:2136",
+ topic_error_probe_seconds=1,
+ )
+
+ if expected_error is None:
+ await _topic_error_probe(config)
+ else:
+ with pytest.raises(expected_error, match="unexpected failure"):
+ await _topic_error_probe(config)
diff --git a/tests/topics/test_topic_writer.py b/tests/topics/test_topic_writer.py
index 035c3b80..f493fe2d 100644
--- a/tests/topics/test_topic_writer.py
+++ b/tests/topics/test_topic_writer.py
@@ -94,6 +94,98 @@ async def test_write_multi_message_with_ack(
assert msg2.seqno == 2
assert msg2.data == "456".encode()
+ async def test_writer_metrics_e2e(self, driver: ydb.aio.Driver, topic_path):
+ from opentelemetry.sdk.metrics import MeterProvider
+ from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+
+ from ydb.observability.metrics import (
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ TOPIC_WRITER_WRITTEN_MESSAGES,
+ )
+ from ydb.opentelemetry import disable_metrics, enable_metrics
+
+ reader = InMemoryMetricReader()
+ provider = MeterProvider(metric_readers=[reader])
+ disable_metrics()
+ enable_metrics(provider)
+ try:
+ async with driver.topic_client.writer(
+ topic_path,
+ codec=ydb.TopicCodec.GZIP,
+ writer_name="e2e-writer",
+ ) as writer:
+ await writer.write_with_ack(
+ [
+ ydb.TopicWriterMessage(data=b"first"),
+ ydb.TopicWriterMessage(data=b"second"),
+ ]
+ )
+ data = reader.get_metrics_data()
+
+ metrics = {
+ metric.name: metric
+ for resource_metrics in data.resource_metrics
+ for scope_metrics in resource_metrics.scope_metrics
+ for metric in scope_metrics.metrics
+ }
+ written_metric = metrics[TOPIC_WRITER_WRITTEN_MESSAGES]
+ written_points = list(written_metric.data.data_points)
+
+ assert written_metric.unit == "{message}"
+ assert len(written_points) == 1
+ assert written_points[0].value == 2
+ assert written_points[0].attributes["topic"] == topic_path
+ assert written_points[0].attributes["writer.name"] == "e2e-writer"
+ assert "status" not in written_points[0].attributes
+
+ processing_metric = metrics[TOPIC_WRITER_MESSAGE_PROCESSING_DURATION]
+ processing_points = list(processing_metric.data.data_points)
+ assert processing_metric.unit == "s"
+ assert len(processing_points) == 1
+ assert processing_points[0].count == 2
+ assert processing_points[0].sum >= 0
+ assert processing_points[0].attributes["topic"] == topic_path
+ assert processing_points[0].attributes["writer.name"] == "e2e-writer"
+ assert processing_points[0].attributes["codec"] == "gzip"
+ assert processing_points[0].attributes["outcome"] == "ok"
+ assert "error.type" not in processing_points[0].attributes
+
+ compression_metric = metrics[TOPIC_WRITER_COMPRESSION_DURATION]
+ compression_points = list(compression_metric.data.data_points)
+ assert compression_metric.unit == "s"
+ assert len(compression_points) == 1
+ assert compression_points[0].count == 2
+ assert compression_points[0].sum >= 0
+ assert compression_points[0].attributes["topic"] == topic_path
+ assert compression_points[0].attributes["writer.name"] == "e2e-writer"
+ assert compression_points[0].attributes["codec"] == "gzip"
+ assert compression_points[0].attributes["outcome"] == "ok"
+ assert "error.type" not in compression_points[0].attributes
+
+ ack_metric = metrics[TOPIC_WRITER_MESSAGE_ACK_DURATION]
+ ack_points = list(ack_metric.data.data_points)
+ assert ack_metric.unit == "s"
+ assert len(ack_points) == 1
+ assert ack_points[0].count == 2
+ assert ack_points[0].sum >= 0
+ assert ack_points[0].attributes["topic"] == topic_path
+ assert ack_points[0].attributes["writer.name"] == "e2e-writer"
+ assert "status" not in ack_points[0].attributes
+
+ oldest_age_metric = metrics[TOPIC_WRITER_SENDING_OLDEST_AGE]
+ oldest_age_points = list(oldest_age_metric.data.data_points)
+ assert oldest_age_metric.unit == "s"
+ assert len(oldest_age_points) == 1
+ assert oldest_age_points[0].value == 0
+ assert oldest_age_points[0].attributes["topic"] == topic_path
+ assert oldest_age_points[0].attributes["writer.name"] == "e2e-writer"
+ finally:
+ disable_metrics()
+ provider.shutdown()
+
@pytest.mark.parametrize(
"codec",
[
diff --git a/ydb/_grpc/grpcwrapper/ydb_topic.py b/ydb/_grpc/grpcwrapper/ydb_topic.py
index 19b38bf0..0de3a749 100644
--- a/ydb/_grpc/grpcwrapper/ydb_topic.py
+++ b/ydb/_grpc/grpcwrapper/ydb_topic.py
@@ -486,11 +486,14 @@ class InitRequest(IToProto):
topics_read_settings: List["StreamReadMessage.InitRequest.TopicReadSettings"]
consumer: Optional[str]
auto_partitioning_support: bool
+ reader_name: Optional[str] = None
def to_proto(self) -> ydb_topic_pb2.StreamReadMessage.InitRequest:
res = ydb_topic_pb2.StreamReadMessage.InitRequest()
if self.consumer is not None:
res.consumer = self.consumer
+ if self.reader_name is not None:
+ res.reader_name = self.reader_name
for settings in self.topics_read_settings:
res.topics_read_settings.append(settings.to_proto())
res.auto_partitioning_support = self.auto_partitioning_support
diff --git a/ydb/_grpc/grpcwrapper/ydb_topic_test.py b/ydb/_grpc/grpcwrapper/ydb_topic_test.py
index b9e30603..5e6ff1c7 100644
--- a/ydb/_grpc/grpcwrapper/ydb_topic_test.py
+++ b/ydb/_grpc/grpcwrapper/ydb_topic_test.py
@@ -2,7 +2,7 @@
from google.protobuf.json_format import MessageToDict
-from ydb._grpc.grpcwrapper.ydb_topic import OffsetsRange
+from ydb._grpc.grpcwrapper.ydb_topic import OffsetsRange, StreamReadMessage
from .ydb_topic import AlterTopicRequest
from .ydb_topic_public_types import (
AlterTopicRequestParams,
@@ -96,3 +96,25 @@ def test_alter_topic_request_from_public_to_proto():
}
assert msg_dict == expected_dict
+
+
+def test_stream_read_init_request_serializes_reader_name():
+ request = StreamReadMessage.InitRequest(
+ topics_read_settings=[],
+ consumer="analytics",
+ auto_partitioning_support=True,
+ reader_name="payments-worker",
+ )
+
+ assert request.to_proto().reader_name == "payments-worker"
+
+
+def test_stream_read_init_request_omits_reader_name_by_default():
+ request = StreamReadMessage.InitRequest(
+ topics_read_settings=[],
+ consumer="analytics",
+ auto_partitioning_support=True,
+ )
+
+ assert request.reader_name is None
+ assert request.to_proto().reader_name == ""
diff --git a/ydb/_topic_reader/datatypes.py b/ydb/_topic_reader/datatypes.py
index 19f706e2..03d25504 100644
--- a/ydb/_topic_reader/datatypes.py
+++ b/ydb/_topic_reader/datatypes.py
@@ -94,22 +94,26 @@ def add_waiter(self, end_offset: int) -> "PartitionSession.CommitAckWaiter":
return waiter
- def ack_notify(self, offset: int):
+ def ack_notify(self, offset: int) -> int:
self._ensure_not_closed()
- self.committed_offset = offset
+ previous_committed_offset = self.committed_offset
+ self.committed_offset = max(previous_committed_offset, offset)
+ acknowledged_messages = self.committed_offset - previous_committed_offset
if not self._ack_waiters:
# todo log warning
# must be never receive ack for not sended request
- return
+ return acknowledged_messages
while self._ack_waiters:
- if self._ack_waiters[0].end_offset > offset:
+ if self._ack_waiters[0].end_offset > self.committed_offset:
break
waiter = self._ack_waiters.popleft()
waiter._finish_ok()
+ return acknowledged_messages
+
def _update_last_commited_offset_if_needed(self, offset: int):
self.committed_offset = max(self.committed_offset, offset)
@@ -168,6 +172,7 @@ class PublicBatch(ICommittable, ISessionAlive):
_partition_session: PartitionSession
_bytes_size: int
_codec: Codec
+ _received_at: Optional[float] = field(default=None, compare=False, repr=False)
def _commit_get_partition_session(self) -> PartitionSession:
return self.messages[0]._commit_get_partition_session()
diff --git a/ydb/_topic_reader/datatypes_test.py b/ydb/_topic_reader/datatypes_test.py
index 19a8bf36..25571127 100644
--- a/ydb/_topic_reader/datatypes_test.py
+++ b/ydb/_topic_reader/datatypes_test.py
@@ -34,7 +34,7 @@ def session(self) -> PartitionSession:
([1], 10, [1], []),
([1, 2, 3], 10, [1, 2, 3], []),
([1, 2, 10, 20], 10, [1, 2, 10], [20]),
- ([10, 20], 1, [], [10, 20]),
+ ([10, 20], 1, [10], [20]),
],
)
async def test_ack_notify(
@@ -66,7 +66,34 @@ def add_notify(future, notified_offset):
notified.sort()
assert notified == offsets_notified
- assert session.committed_offset == notify_offset
+ assert session.committed_offset == max(self.session_comitted_offset, notify_offset)
+
+ async def test_ack_notify_returns_monotonic_committed_offset_advance(self, session):
+ first = session.add_waiter(session.committed_offset + 2)
+ second = session.add_waiter(session.committed_offset + 5)
+
+ assert session.ack_notify(session.committed_offset + 3) == 3
+ assert first.future.done()
+ assert not second.future.done()
+
+ assert session.ack_notify(session.committed_offset + 2) == 2
+ assert second.future.done()
+ assert session.ack_notify(session.committed_offset) == 0
+
+ stale_offset = session.committed_offset - 1
+ assert session.ack_notify(stale_offset) == 0
+ assert session.committed_offset == stale_offset + 1
+
+ async def test_ack_notify_does_not_double_count_overlapping_ranges(self, session):
+ first = session.add_waiter(session.committed_offset + 5)
+ second = session.add_waiter(session.committed_offset + 7)
+
+ assert session.ack_notify(session.committed_offset + 7) == 7
+ assert first.future.done()
+ assert second.future.done()
+
+ async def test_ack_notify_counts_offset_advance_without_waiters(self, session):
+ assert session.ack_notify(session.committed_offset + 2) == 2
# noinspection PyTypeChecker
@pytest.mark.parametrize(
diff --git a/ydb/_topic_reader/topic_reader.py b/ydb/_topic_reader/topic_reader.py
index e4545f3a..59f280d5 100644
--- a/ydb/_topic_reader/topic_reader.py
+++ b/ydb/_topic_reader/topic_reader.py
@@ -60,13 +60,18 @@ class PublicReaderSettings:
buffer_release_threshold: float = 0.5
"""Min fraction of buffer_size_bytes to accumulate before sending a new ReadRequest (0.0 = immediately after every batch)."""
+ reader_name: Optional[str] = None
+ """Optional reader name used to identify this logical reader in diagnostics."""
+
def __post_init__(self):
+ if self.reader_name is not None and not isinstance(self.reader_name, str):
+ raise TypeError("Unsupported type for reader_name field: '%s'" % type(self.reader_name))
if not (0.0 <= self.buffer_release_threshold <= 1.0):
raise ValueError("buffer_release_threshold must be in [0.0, 1.0], got %s" % self.buffer_release_threshold)
# check possible create init message
_ = self._init_message()
- def _init_message(self) -> StreamReadMessage.InitRequest:
+ def _init_message(self, *, reader_name: Optional[str] = None) -> StreamReadMessage.InitRequest:
if self.consumer is not None and not isinstance(self.consumer, str):
raise TypeError("Unsupported type for customer field: '%s'" % type(self.consumer))
@@ -87,6 +92,7 @@ def _init_message(self) -> StreamReadMessage.InitRequest:
topics_read_settings=list(map(PublicTopicSelector._to_topic_read_settings, selectors)), # type: ignore
consumer=self.consumer,
auto_partitioning_support=self.auto_partitioning_support,
+ reader_name=self.reader_name if reader_name is None else reader_name,
)
def _retry_settings(self) -> RetrySettings:
diff --git a/ydb/_topic_reader/topic_reader_asyncio.py b/ydb/_topic_reader/topic_reader_asyncio.py
index 74e9bca7..9a1394f3 100644
--- a/ydb/_topic_reader/topic_reader_asyncio.py
+++ b/ydb/_topic_reader/topic_reader_asyncio.py
@@ -4,6 +4,7 @@
import concurrent.futures
import gzip
import math
+from time import monotonic
import typing
from asyncio import Task
from collections import defaultdict, OrderedDict
@@ -35,6 +36,11 @@
import logging
from ..query.base import TxEvent
+from ..observability.metrics import (
+ TopicReaderMetrics,
+ _NOOP_TOPIC_READER_METRICS,
+ create_topic_reader_metrics,
+)
if typing.TYPE_CHECKING:
from ..query.transaction import BaseQueryTxContext
@@ -43,6 +49,18 @@
logger = logging.getLogger(__name__)
+_CODEC_NAMES = {
+ Codec.CODEC_UNSPECIFIED: "unspecified",
+ Codec.CODEC_RAW: "raw",
+ Codec.CODEC_GZIP: "gzip",
+ Codec.CODEC_LZOP: "lzop",
+ Codec.CODEC_ZSTD: "zstd",
+}
+
+
+def _codec_name(codec: int) -> str:
+ return _CODEC_NAMES.get(codec, str(codec))
+
class TopicReaderError(YdbError):
pass
@@ -87,6 +105,7 @@ def __init__(self):
class PublicAsyncIOReader:
_loop: asyncio.AbstractEventLoop
_closed: bool
+ _log_prefix: str
_settings: topic_reader.PublicReaderSettings
_reconnector: ReaderReconnector
_parent: typing.Any # need for prevent close parent client by GC
@@ -100,8 +119,10 @@ def __init__(
):
self._loop = asyncio.get_running_loop()
self._closed = False
+ self._log_prefix = "topic reader"
self._settings = settings
self._reconnector = ReaderReconnector(driver, settings, self._loop)
+ self._log_prefix = self._reconnector._log_prefix
self._parent = _parent
async def __aenter__(self):
@@ -111,13 +132,19 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.close()
def __del__(self):
- if not self._closed:
- try:
- logger.debug("Topic reader was not closed properly. Consider using method close().")
- task = self._loop.create_task(self.close(flush=False))
- task.set_name("close reader")
- except BaseException:
- logger.warning("Something went wrong during reader close in __del__")
+ if getattr(self, "_closed", True):
+ return
+
+ loop = getattr(self, "_loop", None)
+ if getattr(self, "_reconnector", None) is None or loop is None or loop.is_closed() or not loop.is_running():
+ return
+
+ try:
+ logger.debug("%s was not closed properly. Consider using method close().", self._log_prefix)
+ task = loop.create_task(self.close(flush=False))
+ task.set_name("close reader")
+ except BaseException:
+ logger.warning("%s failed to close in __del__", self._log_prefix)
async def wait_message(self):
"""
@@ -140,7 +167,7 @@ async def receive_batch(
use asyncio.wait_for for wait with timeout.
"""
- logger.debug("receive_batch max_messages=%s max_bytes=%s", max_messages, max_bytes)
+ logger.debug("%s receive_batch max_messages=%s max_bytes=%s", self._log_prefix, max_messages, max_bytes)
await self._reconnector.wait_message()
return self._reconnector.receive_batch_nowait(
max_messages=max_messages,
@@ -163,7 +190,13 @@ async def receive_batch_with_tx(
use asyncio.wait_for for wait with timeout.
"""
- logger.debug("receive_batch_with_tx tx=%s max_messages=%s max_bytes=%s", tx, max_messages, max_bytes)
+ logger.debug(
+ "%s receive_batch_with_tx tx=%s max_messages=%s max_bytes=%s",
+ self._log_prefix,
+ tx,
+ max_messages,
+ max_bytes,
+ )
await self._reconnector.wait_message()
return self._reconnector.receive_batch_with_tx_nowait(
tx=tx,
@@ -177,7 +210,7 @@ async def receive_message(self) -> typing.Optional[datatypes.PublicMessage]:
use asyncio.wait_for for wait with timeout.
"""
- logger.debug("receive_message")
+ logger.debug("%s receive_message", self._log_prefix)
await self._reconnector.wait_message()
return self._reconnector.receive_message_nowait()
@@ -188,7 +221,7 @@ def commit(self, batch: typing.Union[datatypes.PublicMessage, datatypes.PublicBa
For the method no way check the commit result
(for example if lost connection - commits will not re-send and committed messages will receive again).
"""
- logger.debug("commit message or batch")
+ logger.debug("%s commit message or batch", self._log_prefix)
if self._settings.consumer is None:
raise issues.Error("Commit operations are not supported for topic reader without consumer.")
@@ -207,7 +240,7 @@ async def commit_with_ack(self, batch: typing.Union[datatypes.PublicMessage, dat
before receive commit ack. Message may be acked or not (if not - it will send in other read session,
to this or other reader).
"""
- logger.debug("commit_with_ack message or batch")
+ logger.debug("%s commit_with_ack message or batch", self._log_prefix)
if self._settings.consumer is None:
raise issues.Error("Commit operations are not supported for topic reader without consumer.")
@@ -218,10 +251,10 @@ async def close(self, flush: bool = True):
if self._closed:
raise TopicReaderClosedError()
- logger.debug("Close topic reader")
+ logger.debug("%s close", self._log_prefix)
self._closed = True
await self._reconnector.close(flush)
- logger.debug("Topic reader was closed")
+ logger.debug("%s was closed", self._log_prefix)
@property
def read_session_id(self) -> Optional[str]:
@@ -235,6 +268,9 @@ class ReaderReconnector:
_settings: topic_reader.PublicReaderSettings
_driver: Driver
_background_tasks: Set[Task]
+ _reader_name: str
+ _metrics: TopicReaderMetrics
+ _log_prefix: str
_state_changed: asyncio.Event
_stream_reader: Optional["ReaderStream"]
@@ -251,9 +287,17 @@ def __init__(
self._id = ReaderReconnector._static_reader_reconnector_counter.inc_and_get()
self._settings = settings
self._driver = driver
+ self._reader_name = settings.reader_name or "reader-%d" % self._id
+ self._metrics = create_topic_reader_metrics(
+ driver,
+ consumer_name=settings.consumer,
+ reader_name=settings.reader_name,
+ topics=settings.topic,
+ )
+ self._log_prefix = "topic reader reader_name=%r reader_id=%s" % (self._reader_name, self._id)
self._loop = loop if loop is not None else asyncio.get_running_loop()
self._background_tasks = set()
- logger.debug("init reader reconnector id=%s", self._id)
+ logger.debug("%s initialize reconnector", self._log_prefix)
self._state_changed = asyncio.Event()
self._stream_reader = None
@@ -269,21 +313,34 @@ async def _connection_loop(self):
if self._closed:
return
try:
- logger.debug("reader %s connect attempt %s", self._id, attempt)
- self._stream_reader = await ReaderStream.create(self._id, self._driver, self._settings)
- logger.debug("reader %s connected stream %s", self._id, self._stream_reader._id)
+ logger.debug("%s connect attempt=%s", self._log_prefix, attempt)
+ self._stream_reader = await ReaderStream.create(
+ self._id,
+ self._driver,
+ self._settings,
+ reader_name=self._reader_name,
+ metrics=self._metrics,
+ )
+ logger.debug("%s connected stream_id=%s", self._log_prefix, self._stream_reader._id)
attempt = 0
self._state_changed.set()
await self._stream_reader.wait_error()
except BaseException as err:
- logger.debug("reader %s, attempt %s connection loop error %s", self._id, attempt, err)
+ logger.debug("%s connection attempt=%s failed: %s", self._log_prefix, attempt, err)
+ if self._closed:
+ return
retry_info = check_retriable_error(err, self._settings._retry_settings(), attempt)
+ self._metrics.record_session_error(err, retry_info.is_retriable)
if not retry_info.is_retriable:
- logger.debug("reader %s stop connection loop due to %s", self._id, err)
+ logger.debug("%s stop connection loop: %s", self._log_prefix, err)
self._set_first_error(err)
return
- logger.debug("sleep before retry for %s seconds", retry_info.sleep_timeout_seconds)
+ logger.debug(
+ "%s sleep before retry for %s seconds",
+ self._log_prefix,
+ retry_info.sleep_timeout_seconds,
+ )
await asyncio.sleep(retry_info.sleep_timeout_seconds)
@@ -319,10 +376,16 @@ async def wait_message(self):
def receive_batch_nowait(self, max_messages: Optional[int] = None, max_bytes: Optional[int] = None):
if self._stream_reader is None:
return None
- return self._stream_reader.receive_batch_nowait(
+ batch = self._stream_reader.receive_batch_nowait(
max_messages=max_messages,
max_bytes=max_bytes,
)
+ if batch is not None:
+ self._metrics.record_delivered_messages(
+ len(batch.messages),
+ batch._partition_session.topic_path,
+ )
+ return batch
def receive_batch_with_tx_nowait(
self, tx: "BaseQueryTxContext", max_messages: Optional[int] = None, max_bytes: Optional[int] = None
@@ -343,10 +406,21 @@ def receive_batch_with_tx_nowait(
tx._add_callback(TxEvent.AFTER_COMMIT, batch._update_partition_offsets, self._loop)
+ self._metrics.record_delivered_messages(
+ len(batch.messages),
+ batch._partition_session.topic_path,
+ )
+
return batch
def receive_message_nowait(self):
- return self._stream_reader.receive_message_nowait()
+ message = self._stream_reader.receive_message_nowait()
+ if message is not None:
+ self._metrics.record_delivered_messages(
+ 1,
+ message._partition_session.topic_path,
+ )
+ return message
def _init_tx(self, tx: "BaseQueryTxContext"):
tx_id = tx.tx_id
@@ -406,25 +480,31 @@ async def _commit_batches_with_tx(self, tx: "BaseQueryTxContext"):
for topic_path in grouped_batches:
topic_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets(path=topic_path, partitions=[])
for partition_id in grouped_batches[topic_path]:
+ ranges = [batch._commit_get_offsets_range() for batch in grouped_batches[topic_path][partition_id]]
partition_offsets = UpdateOffsetsInTransactionRequest.TopicOffsets.PartitionOffsets(
partition_id=partition_id,
- partition_offsets=[
- batch._commit_get_offsets_range() for batch in grouped_batches[topic_path][partition_id]
- ],
+ partition_offsets=ranges,
)
topic_offsets.partitions.append(partition_offsets)
request.topics.append(topic_offsets)
+ for topic_offsets in request.topics:
+ for partition_offsets in topic_offsets.partitions:
+ for offsets in partition_offsets.partition_offsets:
+ self._metrics.record_commit_queued(offsets.end - offsets.start, topic_offsets.path)
+
try:
- return await self._do_commit_batches_with_tx_call(request)
+ result = await self._do_commit_batches_with_tx_call(request)
except BaseException:
err = issues.ClientInternalError("Failed to update offsets in tx.")
tx._set_external_error(err)
if self._stream_reader is not None:
self._stream_reader._set_first_error(err)
- finally:
if tx_id in self._tx_to_batches_map:
del self._tx_to_batches_map[tx_id]
+ return None
+
+ return result
async def _do_commit_batches_with_tx_call(self, request: UpdateOffsetsInTransactionRequest):
args = [
@@ -451,8 +531,15 @@ async def _handle_after_tx_rollback(self, tx: "BaseQueryTxContext", exc: Optiona
async def _handle_after_tx_commit(self, tx: "BaseQueryTxContext", exc: Optional[BaseException]) -> None:
tx_id = tx.tx_id
- if tx_id is not None and tx_id in self._tx_to_batches_map:
- del self._tx_to_batches_map[tx_id]
+ batches = self._tx_to_batches_map.pop(tx_id, []) if tx_id is not None else []
+
+ if exc is None:
+ for batch in batches:
+ offsets = batch._commit_get_offsets_range()
+ self._metrics.record_commit_acknowledged(
+ offsets.end - offsets.start,
+ batch._partition_session.topic_path,
+ )
if exc is not None and self._stream_reader is not None:
self._stream_reader._set_first_error(
@@ -465,21 +552,27 @@ def commit(self, batch: datatypes.ICommittable) -> datatypes.PartitionSession.Co
return self._stream_reader.commit(batch)
async def close(self, flush: bool):
- logger.debug("reader reconnector %s close", self._id)
+ logger.debug("%s close reconnector", self._log_prefix)
# Mark closed so the connection loop won't start a new stream, then close the
# current stream with the requested flush before cancelling the loop. On a normal
# close this flushes pending commits; cancelling the loop first would let it close
# the stream with flush=False instead and skip the flush.
self._closed = True
- if self._stream_reader:
- await self._stream_reader.close(flush)
- # Wake any pending wait_message() waiter (e.g. a concurrent receive) so it doesn't
- # hang if the loop was reconnecting when close() cancelled it.
- self._set_first_error(TopicReaderStreamClosedError())
- for task in self._background_tasks:
- task.cancel()
-
- await asyncio.wait(self._background_tasks)
+ try:
+ if self._stream_reader:
+ await self._stream_reader.close(flush)
+ finally:
+ try:
+ # Wake any pending wait_message() waiter (e.g. a concurrent receive) so it doesn't
+ # hang if the loop was reconnecting when close() cancelled it.
+ self._set_first_error(TopicReaderStreamClosedError())
+ for task in self._background_tasks:
+ task.cancel()
+
+ if self._background_tasks:
+ await asyncio.wait(self._background_tasks)
+ finally:
+ self._metrics.close()
async def flush(self):
if self._stream_reader:
@@ -506,6 +599,8 @@ class ReaderStream:
_loop: asyncio.AbstractEventLoop
_id: int
_reader_reconnector_id: int
+ _reader_name: str
+ _metrics: TopicReaderMetrics
_session_id: str
_stream: Optional[IGrpcWrapperAsyncIO]
_started: bool
@@ -537,12 +632,18 @@ def __init__(
reader_reconnector_id: int,
settings: topic_reader.PublicReaderSettings,
get_token_function: Optional[Callable[[], str]] = None,
+ *,
+ reader_name: str,
+ metrics: TopicReaderMetrics = _NOOP_TOPIC_READER_METRICS,
):
self._loop = asyncio.get_running_loop()
self._id = ReaderStream._static_id_counter.inc_and_get()
self._reader_reconnector_id = reader_reconnector_id
+ self._reader_name = reader_name
+ self._metrics = metrics
self._session_id = "not initialized"
- self._log_prefix = "reader %s stream %s session=%s" % (
+ self._log_prefix = "topic reader reader_name=%r reader_id=%s stream_id=%s session_id=%s" % (
+ self._reader_name,
self._reader_reconnector_id,
self._id,
self._session_id,
@@ -572,13 +673,16 @@ def __init__(
self._settings = settings
- logger.debug("created ReaderStream id=%s reconnector=%s", self._id, self._reader_reconnector_id)
+ logger.debug("%s created", self._log_prefix)
@staticmethod
async def create(
reader_reconnector_id: int,
driver: SupportedDriverType,
settings: topic_reader.PublicReaderSettings,
+ *,
+ reader_name: str,
+ metrics: TopicReaderMetrics = _NOOP_TOPIC_READER_METRICS,
) -> "ReaderStream":
stream = GrpcWrapperAsyncIO(StreamReadMessage.FromServer.from_proto)
reader = None
@@ -590,8 +694,10 @@ async def create(
reader_reconnector_id,
settings,
get_token_function=creds.get_auth_token if creds else None,
+ reader_name=reader_name,
+ metrics=metrics,
)
- await reader._start(stream, settings._init_message())
+ await reader._start(stream, settings._init_message(reader_name=reader_name))
except BaseException:
# If create() is interrupted (e.g. reader.close() cancels the connection loop
# mid-reconnect) the in-flight stream is not yet assigned to the reconnector, so
@@ -623,7 +729,8 @@ async def _start(self, stream: IGrpcWrapperAsyncIO, init_message: StreamReadMess
if isinstance(init_response.server_message, StreamReadMessage.InitResponse):
self._session_id = init_response.server_message.session_id
- self._log_prefix = "reader %s stream %s session=%s" % (
+ self._log_prefix = "topic reader reader_name=%r reader_id=%s stream_id=%s session_id=%s" % (
+ self._reader_name,
self._reader_reconnector_id,
self._id,
self._session_id,
@@ -689,6 +796,10 @@ def receive_batch_nowait(self, max_messages: Optional[int] = None, max_bytes: Op
part_sess_id, batch = self._get_first_batch()
cutted_batch = batch._pop_batch(max_messages=max_messages, max_bytes=max_bytes)
+ self._metrics.remove_local_buffer_messages(
+ len(cutted_batch.messages),
+ cutted_batch._partition_session,
+ )
if not batch.empty():
self._return_batch_to_queue(part_sess_id, batch)
@@ -708,6 +819,7 @@ def receive_message_nowait(self):
part_sess_id, batch = self._get_first_batch()
message, msgs_left = batch._pop()
+ self._metrics.remove_local_buffer_messages(1, message._partition_session)
if not msgs_left:
self._buffer_release_bytes(batch._bytes_size)
@@ -743,6 +855,11 @@ def commit(self, batch: datatypes.ICommittable) -> datatypes.PartitionSession.Co
)
if self._stream is not None:
self._stream.write(StreamReadMessage.FromClient(client_message=client_message))
+ self._metrics.record_commit_offset_queued(partition_session, commit_range.end)
+ self._metrics.record_commit_queued(
+ commit_range.end - commit_range.start,
+ partition_session.topic_path,
+ )
return waiter
@@ -760,13 +877,15 @@ async def _handle_background_errors(self):
async def _read_messages_loop(self):
try:
logger.debug("%s start read loop", self._log_prefix)
+ initial_credit_bytes = self._buffer_size_bytes
self._stream.write(
StreamReadMessage.FromClient(
client_message=StreamReadMessage.ReadRequest(
- bytes_size=self._buffer_size_bytes,
+ bytes_size=initial_credit_bytes,
),
)
)
+ self._metrics.reset_credit_balance_bytes(initial_credit_bytes)
while True:
try:
message = await self._stream.receive() # type: StreamReadMessage.FromServer
@@ -820,7 +939,7 @@ async def _read_messages_loop(self):
"Unexpected message in _read_messages_loop: %s" % type(message.server_message)
)
except issues.UnexpectedGrpcMessage as e:
- logger.exception("unexpected message in stream reader: %s" % e)
+ logger.exception("%s unexpected message in stream reader: %s", self._log_prefix, e)
self._state_changed.set()
except asyncio.CancelledError as e:
@@ -859,7 +978,7 @@ async def _on_start_partition_session(self, message: StreamReadMessage.StartPart
"Double start partition session: %s" % message.partition_session.partition_session_id
)
- self._partition_sessions[message.partition_session.partition_session_id] = datatypes.PartitionSession(
+ partition_session = datatypes.PartitionSession(
id=message.partition_session.partition_session_id,
state=datatypes.PartitionSession.State.Active,
topic_path=message.partition_session.path,
@@ -868,6 +987,8 @@ async def _on_start_partition_session(self, message: StreamReadMessage.StartPart
reader_reconnector_id=self._reader_reconnector_id,
reader_stream_id=self._id,
)
+ self._partition_sessions[partition_session.id] = partition_session
+ self._metrics.add_partition_session(partition_session.topic_path, partition_session)
read_offset = None
@@ -901,6 +1022,7 @@ def _on_partition_session_stop(self, message: StreamReadMessage.StopPartitionSes
partition = self._partition_sessions.pop(message.partition_session_id)
partition.close()
+ self._metrics.remove_partition_session(partition.topic_path, partition)
if message.graceful and self._stream is not None:
self._stream.write(
@@ -922,35 +1044,51 @@ def _on_end_partition_session(self, message: StreamReadMessage.EndPartitionSessi
self._partition_sessions[message.partition_session_id].end()
def _on_read_response(self, message: StreamReadMessage.ReadResponse):
+ received_at = self._metrics.capture_received_at()
self._buffer_consume_bytes(message.bytes_size)
+ self._metrics.record_received_bytes(message.bytes_size)
batches = self._read_response_to_batches(message)
for batch in batches:
+ batch._received_at = received_at
self._batches_to_decode.put_nowait(batch)
+ self._metrics.record_received_messages(batches, received_at=received_at)
+
def _on_commit_response(self, message: StreamReadMessage.CommitOffsetResponse):
for partition_offset in message.partitions_committed_offsets:
if partition_offset.partition_session_id not in self._partition_sessions:
continue
session = self._partition_sessions[partition_offset.partition_session_id]
- session.ack_notify(partition_offset.committed_offset)
+ acknowledged_messages = session.ack_notify(partition_offset.committed_offset)
+ self._metrics.record_commit_offset_acknowledged(
+ session,
+ partition_offset.committed_offset,
+ )
+ self._metrics.record_commit_acknowledged(
+ acknowledged_messages,
+ session.topic_path,
+ )
def _buffer_consume_bytes(self, bytes_size):
self._buffer_size_bytes -= bytes_size
+ self._metrics.add_credit_balance_bytes(-bytes_size)
def _buffer_release_bytes(self, bytes_size):
self._pending_buffer_release_bytes += bytes_size
if self._pending_buffer_release_bytes >= self._min_buffer_release_bytes:
- self._buffer_size_bytes += self._pending_buffer_release_bytes
+ released_bytes = self._pending_buffer_release_bytes
self._stream.write(
StreamReadMessage.FromClient(
client_message=StreamReadMessage.ReadRequest(
- bytes_size=self._pending_buffer_release_bytes,
+ bytes_size=released_bytes,
)
)
)
+ self._buffer_size_bytes += released_bytes
self._pending_buffer_release_bytes = 0
+ self._metrics.add_credit_balance_bytes(released_bytes)
def _read_response_to_batches(self, message: StreamReadMessage.ReadResponse) -> typing.List[datatypes.PublicBatch]:
batches: typing.List[datatypes.PublicBatch] = []
@@ -1031,19 +1169,66 @@ async def _decode_batch_inplace(self, batch):
try:
decode_func = self._decoders[batch._codec]
except KeyError:
- raise PublicTopicReaderUnexpectedCodecError("Receive message with unexpected codec: %s" % batch._codec)
+ error = PublicTopicReaderUnexpectedCodecError("Receive message with unexpected codec: %s" % batch._codec)
+ if batch._received_at is not None:
+ processing_duration = monotonic() - batch._received_at
+ codec = _codec_name(batch._codec)
+ for _ in batch.messages:
+ self._metrics.record_compression_duration(0.0, codec, batch._partition_session.topic_path, error)
+ self._metrics.record_message_processing_duration(
+ processing_duration,
+ codec,
+ batch._partition_session.topic_path,
+ error,
+ )
+ raise error
decode_data_futures = []
+ if batch._received_at is None:
+ for message in batch.messages:
+ future = self._loop.run_in_executor(self._decode_executor, decode_func, message.data)
+ decode_data_futures.append(future)
+
+ decoded_data = await asyncio.gather(*decode_data_futures)
+ for index, message in enumerate(batch.messages):
+ message.data = decoded_data[index]
+
+ batch._codec = Codec.CODEC_RAW
+ return
+
for message in batch.messages:
- future = self._loop.run_in_executor(self._decode_executor, decode_func, message.data)
+ future = self._loop.run_in_executor(
+ self._decode_executor, self._decode_with_timing, decode_func, message.data
+ )
decode_data_futures.append(future)
decoded_data = await asyncio.gather(*decode_data_futures)
+ processing_duration = monotonic() - batch._received_at
+ processing_error = next((error for _, _, error in decoded_data if error is not None), None)
+ codec = _codec_name(batch._codec)
+ topic = batch._partition_session.topic_path
+
+ for _, compression_duration, error in decoded_data:
+ self._metrics.record_compression_duration(compression_duration, codec, topic, error)
+ self._metrics.record_message_processing_duration(processing_duration, codec, topic, processing_error)
+
+ if processing_error is not None:
+ raise processing_error
+
for index, message in enumerate(batch.messages):
- message.data = decoded_data[index]
+ message.data = decoded_data[index][0]
batch._codec = Codec.CODEC_RAW
+ @staticmethod
+ def _decode_with_timing(decode_func, data):
+ started_at = monotonic()
+ try:
+ decoded = decode_func(data)
+ return decoded, monotonic() - started_at, None
+ except Exception as error:
+ return None, monotonic() - started_at, error
+
def _set_first_error(self, err: YdbError):
try:
self._first_error.set_result(err)
@@ -1078,22 +1263,31 @@ async def close(self, flush: bool):
self._closed = True
logger.debug("%s close", self._log_prefix)
- if flush:
- await self.flush()
-
- self._set_first_error(TopicReaderStreamClosedError())
- self._state_changed.set()
- if self._stream is not None:
- self._stream.close()
-
- for session in self._partition_sessions.values():
- session.close()
- self._partition_sessions.clear()
+ try:
+ if flush:
+ await self.flush()
+ finally:
+ try:
+ self._set_first_error(TopicReaderStreamClosedError())
+ self._state_changed.set()
+ if self._stream is not None:
+ self._stream.close()
- for task in self._background_tasks:
- task.cancel()
+ for session in self._partition_sessions.values():
+ session.close()
+ self._partition_sessions.clear()
- if self._background_tasks:
- await asyncio.wait(self._background_tasks)
+ for task in self._background_tasks:
+ task.cancel()
+ if self._background_tasks:
+ await asyncio.wait(self._background_tasks)
+ finally:
+ try:
+ self._metrics.reset_credit_balance_bytes()
+ finally:
+ try:
+ self._metrics.reset_local_buffer_messages()
+ finally:
+ self._metrics.reset_partition_sessions()
logger.debug("%s was closed", self._log_prefix)
diff --git a/ydb/_topic_reader/topic_reader_asyncio_test.py b/ydb/_topic_reader/topic_reader_asyncio_test.py
index 6d677f77..ec1de76e 100644
--- a/ydb/_topic_reader/topic_reader_asyncio_test.py
+++ b/ydb/_topic_reader/topic_reader_asyncio_test.py
@@ -14,7 +14,12 @@
from . import datatypes, topic_reader_asyncio
from .datatypes import PublicBatch, PublicMessage
from .topic_reader import PublicReaderSettings
-from .topic_reader_asyncio import ReaderStream, ReaderReconnector, TopicReaderError
+from .topic_reader_asyncio import (
+ ReaderStream,
+ ReaderReconnector,
+ TopicReaderError,
+ TopicReaderStreamClosedError,
+)
from .._grpc.grpcwrapper.common_utils import SupportedDriverType, ServerStatus
from .._grpc.grpcwrapper.ydb_topic import (
StreamReadMessage,
@@ -37,6 +42,7 @@
from .._grpc.common.protos import ydb_status_codes_pb2
from .._constants import DEFAULT_INITIAL_RESPONSE_TIMEOUT
+from ..observability.metrics import TopicReaderMetrics
@pytest.fixture(autouse=True)
@@ -88,6 +94,20 @@ def stub_message(id: int):
)
+def test_partially_initialized_public_reader_does_not_schedule_close():
+ reader = object.__new__(topic_reader_asyncio.PublicAsyncIOReader)
+ reader._closed = False
+ reader._log_prefix = "topic reader"
+ reader._loop = mock.Mock()
+ reader._loop.is_closed.return_value = False
+ reader._loop.is_running.return_value = True
+
+ reader.__del__()
+
+ reader._loop.create_task.assert_not_called()
+ reader._closed = True
+
+
@pytest.fixture()
def default_reader_settings(default_executor):
return PublicReaderSettings(
@@ -155,6 +175,7 @@ def second_partition_session(self, default_reader_settings, stream_reader_starte
return stream_reader_started._partition_sessions[partition_session.id]
async def get_started_reader(self, stream, *args, **kwargs) -> ReaderStream:
+ kwargs.setdefault("reader_name", "reader-%d" % self.default_reader_reconnector_id)
reader = ReaderStream(self.default_reader_reconnector_id, *args, **kwargs)
init_message = object()
@@ -182,6 +203,20 @@ async def get_started_reader(self, stream, *args, **kwargs) -> ReaderStream:
return reader
+ async def test_log_prefix_identifies_reader_stream_and_session(self, stream, default_reader_settings):
+ reader = await self.get_started_reader(
+ stream,
+ default_reader_settings,
+ reader_name="payments worker\nprimary",
+ )
+
+ assert reader._log_prefix == (
+ "topic reader reader_name='payments worker\\nprimary' "
+ "reader_id=%s stream_id=%s session_id=test-session" % (self.default_reader_reconnector_id, reader._id)
+ )
+
+ await reader.close(False)
+
@pytest.fixture()
async def stream_reader_started(self, stream, default_reader_settings) -> ReaderStream:
return await self.get_started_reader(stream, default_reader_settings)
@@ -282,6 +317,154 @@ def batch_size():
else:
await wait_condition(lambda: batch_size() > initial_batch_size)
+ async def test_received_metrics_are_recorded_before_decoding(self, stream, default_reader_settings):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ metrics.capture_received_at.return_value = 12.5
+ reader = await self.get_started_reader(
+ stream,
+ default_reader_settings,
+ reader_name="payments-worker",
+ metrics=metrics,
+ )
+ partition_session = datatypes.PartitionSession(
+ id=self.partition_session_id,
+ topic_path=default_reader_settings.topic,
+ partition_id=4,
+ state=datatypes.PartitionSession.State.Active,
+ committed_offset=self.partition_session_committed_offset,
+ reader_reconnector_id=self.default_reader_reconnector_id,
+ reader_stream_id=reader._id,
+ )
+ reader._partition_sessions[partition_session.id] = partition_session
+
+ def assert_bytes_recorded_on_receipt(bytes_size):
+ assert reader._batches_to_decode.qsize() == 0
+ assert partition_session.id not in reader._message_batches
+ assert bytes_size == 1
+
+ def assert_accepted_for_decoding(batches, *, received_at):
+ assert reader._batches_to_decode.qsize() == 1
+ assert partition_session.id not in reader._message_batches
+ assert len(batches) == 1
+ assert len(batches[0].messages) == 2
+ assert batches[0]._partition_session.topic_path == default_reader_settings.topic
+ assert batches[0]._received_at == 12.5
+ assert received_at == 12.5
+
+ metrics.record_received_bytes.side_effect = assert_bytes_recorded_on_receipt
+ metrics.record_received_messages.side_effect = assert_accepted_for_decoding
+
+ try:
+ await self.send_batch(
+ reader,
+ [
+ self.create_message(partition_session, 1, 1),
+ self.create_message(partition_session, 2, 1),
+ ],
+ )
+ metrics.record_received_messages.assert_called_once()
+ metrics.record_received_bytes.assert_called_once_with(1)
+ metrics.add_credit_balance_bytes.assert_called_once_with(-1)
+
+ first = reader.receive_batch_nowait(max_messages=1)
+ second = reader.receive_batch_nowait(max_messages=1)
+ assert len(first.messages) == 1
+ assert len(second.messages) == 1
+ assert metrics.remove_local_buffer_messages.call_args_list == [
+ mock.call(1, partition_session),
+ mock.call(1, partition_session),
+ ]
+ finally:
+ await reader.close(False)
+
+ assert metrics.reset_credit_balance_bytes.call_args_list == [
+ mock.call(default_reader_settings.buffer_size_bytes),
+ mock.call(),
+ ]
+ metrics.reset_local_buffer_messages.assert_called_once_with()
+ metrics.reset_partition_sessions.assert_called_once_with()
+
+ async def test_received_bytes_are_recorded_before_response_conversion(self, default_reader_settings):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reader = ReaderStream(
+ self.default_reader_reconnector_id,
+ default_reader_settings,
+ reader_name="payments-worker",
+ metrics=metrics,
+ )
+ response = StreamReadMessage.ReadResponse(bytes_size=5, partition_data=[])
+
+ with mock.patch.object(
+ reader,
+ "_read_response_to_batches",
+ side_effect=RuntimeError("invalid response"),
+ ):
+ with pytest.raises(RuntimeError, match="invalid response"):
+ reader._on_read_response(response)
+
+ metrics.record_received_bytes.assert_called_once_with(5)
+ metrics.add_credit_balance_bytes.assert_called_once_with(-5)
+ metrics.record_received_messages.assert_not_called()
+
+ async def test_received_bytes_are_recorded_once_for_multitopic_response(self, default_reader_settings):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reader = ReaderStream(
+ self.default_reader_reconnector_id,
+ default_reader_settings,
+ reader_name="payments-worker",
+ metrics=metrics,
+ )
+ topics = ("/Root/a", "/Root/b")
+ for partition_session_id, topic in enumerate(topics, start=1):
+ reader._partition_sessions[partition_session_id] = datatypes.PartitionSession(
+ id=partition_session_id,
+ topic_path=topic,
+ partition_id=partition_session_id,
+ state=datatypes.PartitionSession.State.Active,
+ committed_offset=0,
+ reader_reconnector_id=self.default_reader_reconnector_id,
+ reader_stream_id=reader._id,
+ )
+
+ reader._on_read_response(
+ StreamReadMessage.ReadResponse(
+ bytes_size=5,
+ partition_data=[
+ StreamReadMessage.ReadResponse.PartitionData(
+ partition_session_id=partition_session_id,
+ batches=[
+ StreamReadMessage.ReadResponse.Batch(
+ message_data=[
+ StreamReadMessage.ReadResponse.MessageData(
+ offset=0,
+ seq_no=0,
+ created_at=datetime.datetime(2023, 2, 3, 14, 15),
+ data=b"",
+ uncompresed_size=0,
+ message_group_id="",
+ metadata_items={},
+ )
+ ],
+ producer_id="",
+ write_session_meta={},
+ codec=Codec.CODEC_RAW,
+ written_at=datetime.datetime(2023, 2, 3, 14, 15),
+ )
+ ],
+ )
+ for partition_session_id in reader._partition_sessions
+ ],
+ )
+ )
+
+ metrics.record_received_messages.assert_called_once()
+ batches = metrics.record_received_messages.call_args.args[0]
+ assert [(len(batch.messages), batch._partition_session.topic_path) for batch in batches] == [
+ (1, "/Root/a"),
+ (1, "/Root/b"),
+ ]
+ metrics.record_received_bytes.assert_called_once_with(5)
+
async def test_unknown_error(self, stream, stream_reader_finish_with_error):
class TestError(Exception):
pass
@@ -305,6 +488,13 @@ class TestError(Exception):
),
True,
),
+ (
+ OffsetsRange(
+ partition_session_committed_offset,
+ partition_session_committed_offset + 3,
+ ),
+ True,
+ ),
(
OffsetsRange(
partition_session_committed_offset - 1,
@@ -322,6 +512,9 @@ async def test_send_commit_messages(
commit: OffsetsRange,
send_range: bool,
):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+
@dataclass
class Committable(datatypes.ICommittable):
start: int
@@ -351,15 +544,38 @@ async def wait_message():
]
)
assert partition_session._ack_waiters[-1].end_offset == commit.end
+ metrics.record_commit_queued.assert_called_once_with(
+ commit.end - commit.start,
+ partition_session.topic_path,
+ )
+ metrics.record_commit_offset_queued.assert_called_once_with(partition_session, commit.end)
else:
assert waiter.future.done()
+ metrics.record_commit_queued.assert_not_called()
+ metrics.record_commit_offset_queued.assert_not_called()
with pytest.raises(WaitConditionError):
msg = await wait_message()
pass
assert start_ack_waiters == partition_session._ack_waiters
+ async def test_failed_commit_write_is_not_recorded(self, stream_reader, stream, partition_session):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+ message = self.create_message(partition_session, 1, 1)
+
+ with mock.patch.object(stream, "write", side_effect=RuntimeError("write failed")):
+ with pytest.raises(RuntimeError, match="write failed"):
+ stream_reader.commit(message)
+
+ metrics.record_commit_queued.assert_not_called()
+ metrics.record_commit_offset_queued.assert_not_called()
+ waiter = partition_session._ack_waiters.pop()
+ waiter.future.cancel()
+
async def test_commit_ack_received(self, stream_reader, stream, partition_session, second_partition_session):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
offset1 = self.partition_session_committed_offset + 1
waiter1 = partition_session.add_waiter(offset1)
@@ -386,6 +602,39 @@ async def test_commit_ack_received(self, stream_reader, stream, partition_sessio
await wait_for_fast(waiter1.future)
await wait_for_fast(waiter2.future)
+ assert metrics.record_commit_acknowledged.call_args_list == [
+ mock.call(1, partition_session.topic_path),
+ mock.call(2, second_partition_session.topic_path),
+ ]
+ assert metrics.record_commit_offset_acknowledged.call_args_list == [
+ mock.call(partition_session, offset1),
+ mock.call(second_partition_session, offset2),
+ ]
+
+ stream_reader._on_commit_response(
+ StreamReadMessage.CommitOffsetResponse(
+ partitions_committed_offsets=[
+ StreamReadMessage.CommitOffsetResponse.PartitionCommittedOffset(
+ partition_session_id=partition_session.id,
+ committed_offset=offset1,
+ ),
+ StreamReadMessage.CommitOffsetResponse.PartitionCommittedOffset(
+ partition_session_id=999,
+ committed_offset=100,
+ ),
+ ]
+ )
+ )
+ assert metrics.record_commit_acknowledged.call_args_list == [
+ mock.call(1, partition_session.topic_path),
+ mock.call(2, second_partition_session.topic_path),
+ mock.call(0, partition_session.topic_path),
+ ]
+ assert metrics.record_commit_offset_acknowledged.call_args_list == [
+ mock.call(partition_session, offset1),
+ mock.call(second_partition_session, offset2),
+ mock.call(partition_session, offset1),
+ ]
async def test_close_ack_waiters_when_close_stream_reader(
self, stream_reader_started: ReaderStream, partition_session
@@ -396,6 +645,30 @@ async def test_close_ack_waiters_when_close_stream_reader(
with pytest.raises(topic_reader_asyncio.PublicTopicReaderPartitionExpiredError):
waiter.future.result()
+ @pytest.mark.parametrize(
+ "close_error",
+ [RuntimeError("flush failed"), asyncio.CancelledError()],
+ ids=["error", "cancelled"],
+ )
+ async def test_close_resets_local_buffer_after_flush_failure(self, default_reader_settings, close_error):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reader = ReaderStream(
+ self.default_reader_reconnector_id,
+ default_reader_settings,
+ reader_name="payments-worker",
+ metrics=metrics,
+ )
+ reader.flush = mock.AsyncMock(side_effect=close_error)
+
+ with pytest.raises(type(close_error)):
+ await reader.close(flush=True)
+
+ assert reader._closed
+ assert all(task.done() for task in reader._background_tasks)
+ metrics.reset_credit_balance_bytes.assert_called_once_with()
+ metrics.reset_local_buffer_messages.assert_called_once_with()
+ metrics.reset_partition_sessions.assert_called_once_with()
+
async def test_flush(self, stream, stream_reader_started: ReaderStream, partition_session):
offset = self.partition_session_committed_offset + 1
waiter = partition_session.add_waiter(offset)
@@ -609,6 +882,106 @@ async def test_decode_loop(self, stream_reader, batch: PublicBatch, data_out: ty
assert batch == expected
+ async def test_decode_records_compression_and_processing_duration(self, stream_reader):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+ partition_session = stub_partition_session()
+ partition_session.topic_path = "/Root/orders"
+ message = stub_message(1)
+ message._partition_session = partition_session
+ message.data = gzip.compress(b"payload")
+ batch = PublicBatch(
+ messages=[message],
+ _partition_session=partition_session,
+ _bytes_size=len(message.data),
+ _codec=Codec.CODEC_GZIP,
+ _received_at=10.0,
+ )
+
+ with mock.patch.object(topic_reader_asyncio, "monotonic", side_effect=[11.0, 11.25, 11.5]):
+ await stream_reader._decode_batch_inplace(batch)
+
+ assert message.data == b"payload"
+ assert batch._codec == Codec.CODEC_RAW
+ metrics.record_compression_duration.assert_called_once_with(0.25, "gzip", "/Root/orders", None)
+ metrics.record_message_processing_duration.assert_called_once_with(1.5, "gzip", "/Root/orders", None)
+
+ async def test_decode_failure_records_both_durations_and_is_rethrown(self, stream_reader):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+ partition_session = stub_partition_session()
+ partition_session.topic_path = "/Root/orders"
+ message = stub_message(1)
+ message._partition_session = partition_session
+ problem = RuntimeError("decode failed")
+
+ def fail(_):
+ raise problem
+
+ stream_reader._decoders[Codec.CODEC_GZIP] = fail
+ batch = PublicBatch(
+ messages=[message],
+ _partition_session=partition_session,
+ _bytes_size=10,
+ _codec=Codec.CODEC_GZIP,
+ _received_at=20.0,
+ )
+
+ with mock.patch.object(topic_reader_asyncio, "monotonic", side_effect=[21.0, 21.25, 21.5]):
+ with pytest.raises(RuntimeError, match="decode failed") as raised:
+ await stream_reader._decode_batch_inplace(batch)
+
+ assert raised.value is problem
+ assert batch._codec == Codec.CODEC_GZIP
+ metrics.record_compression_duration.assert_called_once_with(0.25, "gzip", "/Root/orders", problem)
+ metrics.record_message_processing_duration.assert_called_once_with(1.5, "gzip", "/Root/orders", problem)
+
+ async def test_unknown_codec_records_each_message_as_failed(self, stream_reader):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+ partition_session = stub_partition_session()
+ partition_session.topic_path = "/Root/orders"
+ messages = [stub_message(1), stub_message(2)]
+ for message in messages:
+ message._partition_session = partition_session
+ batch = PublicBatch(
+ messages=messages,
+ _partition_session=partition_session,
+ _bytes_size=10,
+ _codec=10001,
+ _received_at=30.0,
+ )
+
+ with mock.patch.object(topic_reader_asyncio, "monotonic", return_value=31.0):
+ with pytest.raises(topic_reader_asyncio.PublicTopicReaderUnexpectedCodecError) as raised:
+ await stream_reader._decode_batch_inplace(batch)
+
+ error = raised.value
+ assert metrics.record_compression_duration.call_args_list == [
+ mock.call(0.0, "10001", "/Root/orders", error),
+ mock.call(0.0, "10001", "/Root/orders", error),
+ ]
+ assert metrics.record_message_processing_duration.call_args_list == [
+ mock.call(1.0, "10001", "/Root/orders", error),
+ mock.call(1.0, "10001", "/Root/orders", error),
+ ]
+
+ async def test_raw_decode_does_not_record_processing_histograms(self, stream_reader):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+ batch = PublicBatch(
+ messages=[stub_message(1)],
+ _partition_session=stub_partition_session(),
+ _bytes_size=1,
+ _codec=Codec.CODEC_RAW,
+ _received_at=10.0,
+ )
+
+ await stream_reader._decode_batch_inplace(batch)
+
+ metrics.record_compression_duration.assert_not_called()
+ metrics.record_message_processing_duration.assert_not_called()
+
async def test_error_from_status_code(self, stream, stream_reader_finish_with_error):
# noinspection PyTypeChecker
stream.from_server.put_nowait(
@@ -628,7 +1001,11 @@ async def test_error_from_status_code(self, stream, stream_reader_finish_with_er
stream_reader_finish_with_error.receive_batch_nowait()
async def test_init_reader(self, stream, default_reader_settings):
- reader = ReaderStream(self.default_reader_reconnector_id, default_reader_settings)
+ reader = ReaderStream(
+ self.default_reader_reconnector_id,
+ default_reader_settings,
+ reader_name="reader-%d" % self.default_reader_reconnector_id,
+ )
init_message = StreamReadMessage.InitRequest(
consumer="test-consumer",
topics_read_settings=[
@@ -671,6 +1048,9 @@ async def test_start_partition(
default_reader_settings,
partition_session,
):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+
def session_count():
return len(stream_reader._partition_sessions)
@@ -717,8 +1097,15 @@ def session_count():
reader_reconnector_id=self.default_reader_reconnector_id,
reader_stream_id=stream_reader._id,
)
+ metrics.add_partition_session.assert_called_once_with(
+ test_topic_path,
+ stream_reader._partition_sessions[test_partition_session_id],
+ )
async def test_partition_stop_force(self, stream, stream_reader, partition_session):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+
def session_count():
return len(stream_reader._partition_sessions)
@@ -741,8 +1128,12 @@ def session_count():
assert session_count() == initial_session_count - 1
assert partition_session.id not in stream_reader._partition_sessions
+ metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path, partition_session)
async def test_partition_stop_graceful(self, stream, stream_reader, partition_session):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+
def session_count():
return len(stream_reader._partition_sessions)
@@ -765,6 +1156,7 @@ def session_count():
assert resp.client_message == StreamReadMessage.StopPartitionSessionResponse(
partition_session_id=partition_session.id
)
+ metrics.remove_partition_session.assert_called_once_with(partition_session.topic_path, partition_session)
stream.from_server.put_nowait(
StreamReadMessage.FromServer(
@@ -782,6 +1174,9 @@ def session_count():
stream.from_client.get_nowait()
async def test_end_partition_session(self, stream, stream_reader, partition_session):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+
def session_count():
return len(stream_reader._partition_sessions)
@@ -805,6 +1200,7 @@ def session_count():
assert session_count() == initial_session_count
assert partition_session.id in stream_reader._partition_sessions
assert partition_session.ended
+ metrics.remove_partition_session.assert_not_called()
@pytest.mark.parametrize(
"graceful",
@@ -1468,6 +1864,31 @@ async def test_receive_partial_batch_releases_only_cut_bytes(self, stream_reader
assert len(remainder.messages) == 2
assert remainder._bytes_size == 60 # remainder stays buffered
+ async def test_local_buffer_is_updated_before_credit_release_failure(self, stream_reader, monkeypatch):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
+ partition_session = stub_partition_session()
+ stream_reader._message_batches = OrderedDict(
+ {
+ 0: PublicBatch(
+ messages=[stub_message(1), stub_message(2)],
+ _partition_session=partition_session,
+ _bytes_size=20,
+ _codec=Codec.CODEC_RAW,
+ )
+ }
+ )
+ monkeypatch.setattr(
+ stream_reader,
+ "_buffer_release_bytes",
+ mock.Mock(side_effect=RuntimeError("credit release failed")),
+ )
+
+ with pytest.raises(RuntimeError, match="credit release failed"):
+ stream_reader.receive_batch_nowait(max_messages=1)
+
+ metrics.remove_local_buffer_messages.assert_called_once_with(1, partition_session)
+
async def test_receive_batch_nowait(self, stream, stream_reader, partition_session):
assert stream_reader.receive_batch_nowait() is None
@@ -1496,6 +1917,8 @@ async def test_receive_batch_nowait(self, stream, stream_reader, partition_sessi
async def test_receive_message_nowait(self, stream, stream_reader, partition_session):
assert stream_reader.receive_batch_nowait() is None
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream_reader._metrics = metrics
initial_buffer_size = stream_reader._buffer_size_bytes
@@ -1520,6 +1943,7 @@ async def test_receive_message_nowait(self, stream, stream_reader, partition_ses
mess = stream_reader.receive_message_nowait()
assert mess.seqno == expected_seqno
+ assert metrics.remove_local_buffer_messages.call_args_list == [mock.call(1, partition_session)] * 3
assert stream_reader._buffer_size_bytes == initial_buffer_size - 2 * self.default_batch_size
async def test_update_token(self, stream):
@@ -1576,7 +2000,11 @@ def logged():
async def test_init_timeout_parameter(self, stream, default_reader_settings):
"""Test that ReaderStream._start calls stream.receive with timeout=10"""
- reader = ReaderStream(self.default_reader_reconnector_id, default_reader_settings)
+ reader = ReaderStream(
+ self.default_reader_reconnector_id,
+ default_reader_settings,
+ reader_name="reader-%d" % self.default_reader_reconnector_id,
+ )
init_message = default_reader_settings._init_message()
# Mock stream.receive to check if timeout is passed
@@ -1595,7 +2023,11 @@ async def test_init_timeout_parameter(self, stream, default_reader_settings):
async def test_init_timeout_behavior(self, stream, default_reader_settings):
"""Test that ReaderStream._start raises TopicReaderError when receive times out"""
- reader = ReaderStream(self.default_reader_reconnector_id, default_reader_settings)
+ reader = ReaderStream(
+ self.default_reader_reconnector_id,
+ default_reader_settings,
+ reader_name="reader-%d" % self.default_reader_reconnector_id,
+ )
init_message = default_reader_settings._init_message()
# Mock stream.receive to directly raise TimeoutError when called with timeout
@@ -1617,8 +2049,12 @@ async def timeout_receive(timeout=None):
@pytest.mark.asyncio
class TestReaderReconnector:
- async def test_reconnect_on_repeatable_error(self, monkeypatch):
+ @pytest.mark.parametrize("configured_reader_name", [None, "", "payments-worker"])
+ async def test_reconnect_on_repeatable_error(self, monkeypatch, configured_reader_name):
test_error = issues.Overloaded("test error")
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ create_metrics = mock.Mock(return_value=metrics)
+ monkeypatch.setattr(topic_reader_asyncio, "create_topic_reader_metrics", create_metrics)
async def wait_error():
raise test_error
@@ -1647,8 +2083,13 @@ async def stream_create(
reader_reconnector_id: int,
driver: SupportedDriverType,
settings: PublicReaderSettings,
+ *,
+ reader_name: str,
+ metrics: TopicReaderMetrics,
):
nonlocal stream_index
+ reader_names.append(reader_name)
+ reader_metrics.append(metrics)
stream_index += 1
if stream_index == 1:
return reader_stream_mock_with_error
@@ -1657,12 +2098,294 @@ async def stream_create(
else:
raise Exception("unexpected create stream")
+ reader_names = []
+ reader_metrics = []
with mock.patch.object(ReaderStream, "create", stream_create):
- reconnector = ReaderReconnector(mock.Mock(), PublicReaderSettings("", ""))
+ reconnector = ReaderReconnector(
+ mock.Mock(),
+ PublicReaderSettings("", "", reader_name=configured_reader_name),
+ )
await wait_for_fast(reconnector.wait_message())
+ assert reader_names == [reconnector._reader_name, reconnector._reader_name]
+ assert reader_metrics == [metrics, metrics]
+ assert reconnector._metrics is metrics
+ expected_reader_name = configured_reader_name or "reader-%d" % reconnector._id
+ assert reconnector._reader_name == expected_reader_name
+ create_metrics.assert_called_once_with(
+ reconnector._driver,
+ consumer_name="",
+ reader_name=configured_reader_name,
+ topics=reconnector._settings.topic,
+ )
reader_stream_mock_with_error.wait_error.assert_any_await()
reader_stream_mock_with_error.wait_messages.assert_any_await()
+ metrics.record_session_error.assert_called_once_with(test_error, True)
+ await reconnector.close(flush=False)
+ metrics.close.assert_called_once_with()
+
+ async def test_non_retriable_connection_error_records_stop(self):
+ test_error = issues.BadRequest("test error")
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._id = 1
+ reconnector._driver = mock.Mock()
+ reconnector._settings = PublicReaderSettings("", "", reader_name="reader")
+ reconnector._reader_name = "reader"
+ reconnector._metrics = metrics
+ reconnector._log_prefix = "test-reader"
+ reconnector._closed = False
+ reconnector._stream_reader = None
+ reconnector._first_error = asyncio.get_running_loop().create_future()
+ reconnector._state_changed = asyncio.Event()
+
+ with mock.patch.object(ReaderStream, "create", mock.AsyncMock(side_effect=test_error)):
+ await reconnector._connection_loop()
+
+ metrics.record_session_error.assert_called_once_with(test_error, False)
+ assert reconnector._first_error.result() is test_error
+
+ async def test_connection_error_during_close_is_not_recorded(self):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._id = 1
+ reconnector._driver = mock.Mock()
+ reconnector._settings = PublicReaderSettings("", "", reader_name="reader")
+ reconnector._reader_name = "reader"
+ reconnector._metrics = metrics
+ reconnector._log_prefix = "test-reader"
+ reconnector._closed = False
+ reconnector._stream_reader = None
+
+ async def close_during_create(*args, **kwargs):
+ reconnector._closed = True
+ raise TopicReaderStreamClosedError()
+
+ with mock.patch.object(ReaderStream, "create", close_during_create):
+ await reconnector._connection_loop()
+
+ metrics.record_session_error.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "close_error",
+ [RuntimeError("stream close failed"), asyncio.CancelledError()],
+ ids=["error", "cancelled"],
+ )
+ async def test_close_unregisters_metrics_after_stream_close_failure(self, close_error):
+ stream = mock.Mock(spec=ReaderStream)
+ stream.close = mock.AsyncMock(side_effect=close_error)
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._log_prefix = "test-reader"
+ reconnector._closed = False
+ reconnector._stream_reader = stream
+ reconnector._first_error = asyncio.get_running_loop().create_future()
+ reconnector._state_changed = asyncio.Event()
+ reconnector._background_tasks = set()
+ reconnector._metrics = metrics
+
+ with pytest.raises(type(close_error)):
+ await reconnector.close(flush=False)
+
+ assert reconnector._closed
+ metrics.close.assert_called_once_with()
+
+ async def test_receive_batch_records_only_returned_messages(self):
+ partition_session = stub_partition_session()
+ partition_session.topic_path = "/Root/orders"
+ batch = PublicBatch(
+ messages=[stub_message(1), stub_message(2)],
+ _partition_session=partition_session,
+ _bytes_size=20,
+ _codec=Codec.CODEC_RAW,
+ )
+ stream = mock.Mock(spec=ReaderStream)
+ stream.receive_batch_nowait.side_effect = [batch, None]
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._stream_reader = stream
+ reconnector._metrics = metrics
+
+ assert reconnector.receive_batch_nowait(max_messages=2, max_bytes=100) is batch
+ assert reconnector.receive_batch_nowait() is None
+
+ metrics.record_delivered_messages.assert_called_once_with(2, "/Root/orders")
+
+ async def test_receive_message_records_only_returned_message(self):
+ message = stub_message(1)
+ message._partition_session.topic_path = "/Root/orders"
+ stream = mock.Mock(spec=ReaderStream)
+ stream.receive_message_nowait.side_effect = [message, None]
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._stream_reader = stream
+ reconnector._metrics = metrics
+
+ assert reconnector.receive_message_nowait() is message
+ assert reconnector.receive_message_nowait() is None
+
+ metrics.record_delivered_messages.assert_called_once_with(1, "/Root/orders")
+
+ async def test_receive_batch_with_tx_records_after_successful_delivery(self):
+ partition_session = stub_partition_session()
+ partition_session.topic_path = "/Root/orders"
+ batch = PublicBatch(
+ messages=[stub_message(1), stub_message(2), stub_message(3)],
+ _partition_session=partition_session,
+ _bytes_size=30,
+ _codec=Codec.CODEC_RAW,
+ )
+ stream = mock.Mock(spec=ReaderStream)
+ stream.receive_batch_nowait.return_value = batch
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ tx = mock.Mock()
+ tx.tx_id = "tx-1"
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._stream_reader = stream
+ reconnector._metrics = metrics
+ reconnector._tx_to_batches_map = {}
+ reconnector._loop = asyncio.get_running_loop()
+
+ result = reconnector.receive_batch_with_tx_nowait(tx, max_messages=3)
+
+ assert result is batch
+ assert reconnector._tx_to_batches_map["tx-1"] == [batch]
+ metrics.record_delivered_messages.assert_called_once_with(3, "/Root/orders")
+
+ async def test_receive_batch_with_tx_does_not_record_failed_delivery(self):
+ partition_session = stub_partition_session()
+ batch = PublicBatch(
+ messages=[stub_message(1)],
+ _partition_session=partition_session,
+ _bytes_size=10,
+ _codec=Codec.CODEC_RAW,
+ )
+ stream = mock.Mock(spec=ReaderStream)
+ stream.receive_batch_nowait.return_value = batch
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ tx = mock.Mock()
+ tx.tx_id = "tx-1"
+ tx._add_callback.side_effect = [None, None, None, RuntimeError("callback failed")]
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._stream_reader = stream
+ reconnector._metrics = metrics
+ reconnector._tx_to_batches_map = {}
+ reconnector._loop = asyncio.get_running_loop()
+
+ with pytest.raises(RuntimeError, match="callback failed"):
+ reconnector.receive_batch_with_tx_nowait(tx)
+
+ metrics.record_delivered_messages.assert_not_called()
+
+ @pytest.mark.parametrize("rpc_error", [None, RuntimeError("rpc failed")], ids=["success", "rpc-failure"])
+ async def test_transaction_commit_records_queued_ranges(self, rpc_error):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream = mock.Mock(spec=ReaderStream)
+ stream._id = 513
+ stream._partition_sessions = {}
+
+ def batch(topic, partition_session_id, partition_id, start, end):
+ partition_session = stub_partition_session(partition_session_id)
+ partition_session.topic_path = topic
+ partition_session.partition_id = partition_id
+ stream._partition_sessions[partition_session_id] = partition_session
+ result = mock.Mock(spec=PublicBatch)
+ result._partition_session = partition_session
+ result._commit_get_offsets_range.return_value = OffsetsRange(start, end)
+ return result
+
+ batches = [
+ batch("/Root/orders", 1, 10, 5, 7),
+ batch("/Root/orders", 2, 20, 11, 14),
+ batch("/Root/refunds", 3, 30, 20, 24),
+ ]
+ tx = mock.Mock()
+ tx.tx_id = "tx-1"
+
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._settings = PublicReaderSettings(consumer="analytics", topic="/Root/orders")
+ reconnector._stream_reader = stream
+ reconnector._tx_to_batches_map = {tx.tx_id: batches}
+ reconnector._metrics = metrics
+ reconnector._do_commit_batches_with_tx_call = mock.AsyncMock(side_effect=rpc_error)
+
+ await reconnector._commit_batches_with_tx(tx)
+
+ assert metrics.record_commit_queued.call_args_list == [
+ mock.call(2, "/Root/orders"),
+ mock.call(3, "/Root/orders"),
+ mock.call(4, "/Root/refunds"),
+ ]
+ metrics.record_commit_offset_queued.assert_not_called()
+ reconnector._do_commit_batches_with_tx_call.assert_awaited_once()
+ if rpc_error is None:
+ assert reconnector._tx_to_batches_map[tx.tx_id] == batches
+ else:
+ assert tx.tx_id not in reconnector._tx_to_batches_map
+
+ async def test_expired_transaction_commit_is_not_recorded(self):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ stream = mock.Mock(spec=ReaderStream)
+ stream._id = 514
+ stream._partition_sessions = {}
+ partition_session = stub_partition_session(1)
+ batch = mock.Mock(spec=PublicBatch)
+ batch._partition_session = partition_session
+ batch._commit_get_offsets_range.return_value = OffsetsRange(5, 7)
+ tx = mock.Mock()
+ tx.tx_id = "tx-1"
+
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._settings = PublicReaderSettings(consumer="analytics", topic="/Root/orders")
+ reconnector._stream_reader = stream
+ reconnector._tx_to_batches_map = {tx.tx_id: [batch]}
+ reconnector._metrics = metrics
+ reconnector._do_commit_batches_with_tx_call = mock.AsyncMock()
+
+ await reconnector._commit_batches_with_tx(tx)
+
+ metrics.record_commit_queued.assert_not_called()
+ metrics.record_commit_offset_queued.assert_not_called()
+ reconnector._do_commit_batches_with_tx_call.assert_not_awaited()
+
+ @pytest.mark.parametrize("commit_error", [None, RuntimeError("commit failed")], ids=["success", "failure"])
+ async def test_transaction_commit_records_acknowledged_ranges_only_on_success(self, commit_error):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+
+ def batch(topic, start, end):
+ result = mock.Mock(spec=PublicBatch)
+ result._partition_session = stub_partition_session(start)
+ result._partition_session.topic_path = topic
+ result._commit_get_offsets_range.return_value = OffsetsRange(start, end)
+ return result
+
+ tx = mock.Mock()
+ tx.tx_id = "tx-1"
+ batches = [
+ batch("/Root/orders", 5, 7),
+ batch("/Root/orders", 11, 14),
+ batch("/Root/refunds", 20, 24),
+ ]
+ reconnector = object.__new__(ReaderReconnector)
+ reconnector._tx_to_batches_map = {tx.tx_id: batches}
+ reconnector._metrics = metrics
+ reconnector._stream_reader = mock.Mock(spec=ReaderStream)
+
+ await reconnector._handle_after_tx_commit(tx, commit_error)
+
+ assert tx.tx_id not in reconnector._tx_to_batches_map
+ if commit_error is None:
+ assert metrics.record_commit_acknowledged.call_args_list == [
+ mock.call(2, "/Root/orders"),
+ mock.call(3, "/Root/orders"),
+ mock.call(4, "/Root/refunds"),
+ ]
+ metrics.record_commit_offset_acknowledged.assert_not_called()
+ reconnector._stream_reader._set_first_error.assert_not_called()
+ else:
+ metrics.record_commit_acknowledged.assert_not_called()
+ metrics.record_commit_offset_acknowledged.assert_not_called()
+ reconnector._stream_reader._set_first_error.assert_called_once()
async def test_close_during_reconnect_does_not_hang(self):
# The connection loop must stop on reader.close() even while it is closing the old
@@ -1695,7 +2418,7 @@ async def wait_forever():
create_calls = 0
- async def stream_create(reader_reconnector_id, driver, settings):
+ async def stream_create(reader_reconnector_id, driver, settings, *, reader_name, metrics):
nonlocal create_calls
create_calls += 1
return stream1 if create_calls == 1 else stream2
@@ -1729,7 +2452,14 @@ async def start(self, driver, stub, method):
with mock.patch.object(topic_reader_asyncio, "GrpcWrapperAsyncIO", FakeStream):
# Real create(); no InitResponse is sent, so it parks inside _start() on
# `await stream.receive()` (the only reachable cancellation point in create()).
- create_task = asyncio.create_task(ReaderStream.create(7, driver, default_reader_settings))
+ create_task = asyncio.create_task(
+ ReaderStream.create(
+ 7,
+ driver,
+ default_reader_settings,
+ reader_name="reader-7",
+ )
+ )
await wait_condition(lambda: bool(built) and not built[0].from_client.empty())
assert not create_task.done()
@@ -1759,7 +2489,7 @@ async def receive(timeout=None):
stream.write = mock.Mock()
stream.close = mock.Mock()
- reader = ReaderStream(4, default_reader_settings)
+ reader = ReaderStream(4, default_reader_settings, reader_name="reader-4")
await reader._start(stream, default_reader_settings._init_message())
# Bug: wait_error() hangs forever because _first_error is never set.
@@ -1785,7 +2515,14 @@ class TestReaderStreamBufferReleaseThreshold:
def stream(self):
return StreamMock()
- async def _get_started_reader(self, stream, threshold, buffer_size_bytes, default_executor) -> ReaderStream:
+ async def _get_started_reader(
+ self,
+ stream,
+ threshold,
+ buffer_size_bytes,
+ default_executor,
+ metrics=None,
+ ) -> ReaderStream:
settings = PublicReaderSettings(
consumer="test-consumer",
topic="test-topic",
@@ -1793,7 +2530,15 @@ async def _get_started_reader(self, stream, threshold, buffer_size_bytes, defaul
buffer_release_threshold=threshold,
decoder_executor=default_executor,
)
- reader = ReaderStream(self.default_reader_reconnector_id, settings)
+ reader_kwargs = {}
+ if metrics is not None:
+ reader_kwargs["metrics"] = metrics
+ reader = ReaderStream(
+ self.default_reader_reconnector_id,
+ settings,
+ reader_name="reader-%d" % self.default_reader_reconnector_id,
+ **reader_kwargs,
+ )
init_message = object()
start = asyncio.create_task(reader._start(stream, init_message))
@@ -1854,6 +2599,49 @@ async def test_threshold_zero_sends_immediately(self, stream, default_executor):
await reader.close(False)
+ async def test_credit_balance_tracks_granted_consumed_and_released_bytes(self, stream, default_executor):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reader = await self._get_started_reader(
+ stream,
+ threshold=0.1,
+ buffer_size_bytes=1000,
+ default_executor=default_executor,
+ metrics=metrics,
+ )
+
+ metrics.reset_credit_balance_bytes.assert_called_once_with(1000)
+
+ reader._buffer_consume_bytes(200)
+ reader._buffer_release_bytes(50)
+ metrics.add_credit_balance_bytes.assert_called_once_with(-200)
+
+ reader._buffer_release_bytes(50)
+ assert metrics.add_credit_balance_bytes.call_args_list == [mock.call(-200), mock.call(100)]
+ request = stream.from_client.get_nowait()
+ assert request.client_message.bytes_size == 100
+
+ await reader.close(False)
+ assert metrics.reset_credit_balance_bytes.call_args_list == [mock.call(1000), mock.call()]
+
+ async def test_credit_balance_is_not_replenished_when_read_request_fails(self, stream, default_executor):
+ metrics = mock.Mock(spec=TopicReaderMetrics)
+ reader = await self._get_started_reader(
+ stream,
+ threshold=0.1,
+ buffer_size_bytes=1000,
+ default_executor=default_executor,
+ metrics=metrics,
+ )
+ stream.write = mock.Mock(side_effect=RuntimeError("write failed"))
+
+ with pytest.raises(RuntimeError, match="write failed"):
+ reader._buffer_release_bytes(100)
+
+ metrics.add_credit_balance_bytes.assert_not_called()
+ assert reader._buffer_size_bytes == 1000
+
+ await reader.close(False)
+
async def test_threshold_delays_release_while_messages_pending(self, stream, default_executor):
"""threshold=0.5: releases below threshold are not flushed, even when queue becomes empty."""
reader = await self._get_started_reader(
@@ -1957,3 +2745,78 @@ async def test_threshold_one_flushes_when_bytes_match_buffer_size(self, stream,
assert msg.client_message.bytes_size == 1000
await reader.close(False)
+
+
+def test_reader_settings_forward_reader_name():
+ settings = PublicReaderSettings(
+ consumer="analytics",
+ topic="/Root/events",
+ reader_name="payments-worker",
+ )
+
+ assert settings._init_message().reader_name == "payments-worker"
+ assert settings._init_message(reader_name="reader-42").to_proto().reader_name == "reader-42"
+
+
+def test_reader_settings_reject_non_string_reader_name():
+ with pytest.raises(TypeError, match="reader_name"):
+ PublicReaderSettings(
+ consumer="analytics",
+ topic="/Root/events",
+ reader_name=42,
+ )
+
+
+@pytest.mark.asyncio
+async def test_reader_reconnector_uses_configured_or_process_local_reader_name():
+ async def wait_forever(self):
+ await asyncio.Future()
+
+ with mock.patch.object(ReaderReconnector, "_connection_loop", wait_forever):
+ generated_from_none = ReaderReconnector(
+ mock.Mock(),
+ PublicReaderSettings("analytics", "/Root/events"),
+ )
+ generated_from_empty = ReaderReconnector(
+ mock.Mock(),
+ PublicReaderSettings("analytics", "/Root/events", reader_name=""),
+ )
+ configured = ReaderReconnector(
+ mock.Mock(),
+ PublicReaderSettings("analytics", "/Root/events", reader_name="payments-worker"),
+ )
+
+ assert generated_from_none._reader_name == "reader-%d" % generated_from_none._id
+ assert generated_from_empty._reader_name == "reader-%d" % generated_from_empty._id
+ assert generated_from_none._reader_name != generated_from_empty._reader_name
+ assert configured._reader_name == "payments-worker"
+ assert configured._log_prefix == "topic reader reader_name='payments-worker' reader_id=%s" % configured._id
+
+ await asyncio.gather(
+ generated_from_none.close(flush=False),
+ generated_from_empty.close(flush=False),
+ configured.close(flush=False),
+ )
+
+
+@pytest.mark.parametrize(
+ "client_class, reader_class_name", [("TopicClient", "TopicReader"), ("TopicClientAsyncIO", "TopicReaderAsyncIO")]
+)
+def test_topic_clients_forward_reader_name(client_class, reader_class_name):
+ from .. import topic as topic_module
+
+ client = object.__new__(getattr(topic_module, client_class))
+ client._closed = False
+ client._driver = mock.Mock()
+ client._executor = mock.Mock()
+
+ with mock.patch.object(topic_module, reader_class_name, return_value=mock.sentinel.reader) as reader_class:
+ result = client.reader(
+ "/Root/events",
+ consumer="analytics",
+ reader_name="payments-worker",
+ )
+
+ assert result is mock.sentinel.reader
+ assert reader_class.call_args.args[1].reader_name == "payments-worker"
+ client._closed = True
diff --git a/ydb/_topic_reader/topic_reader_sync.py b/ydb/_topic_reader/topic_reader_sync.py
index a2a24fef..4a09ebdf 100644
--- a/ydb/_topic_reader/topic_reader_sync.py
+++ b/ydb/_topic_reader/topic_reader_sync.py
@@ -32,6 +32,7 @@ class TopicReaderSync:
_caller: CallFromSyncToAsync
_async_reader: PublicAsyncIOReader
_closed: bool
+ _log_prefix: str
_settings: PublicReaderSettings
_parent: typing.Any # need for prevent stop the client by GC
@@ -44,6 +45,7 @@ def __init__(
_parent=None, # need for prevent stop the client by GC
):
self._closed = False
+ self._log_prefix = "topic reader"
if eventloop:
loop = eventloop
@@ -56,6 +58,7 @@ async def create_reader():
return PublicAsyncIOReader(driver, settings)
self._async_reader = asyncio.run_coroutine_threadsafe(create_reader(), loop).result()
+ self._log_prefix = self._async_reader._log_prefix
self._settings = settings
@@ -64,10 +67,10 @@ async def create_reader():
def __del__(self):
if not self._closed:
try:
- logger.debug("Topic reader was not closed properly. Consider using method close().")
+ logger.debug("%s was not closed properly. Consider using method close().", self._log_prefix)
self.close(flush=False)
except BaseException:
- logger.warning("Something went wrong during reader close in __del__")
+ logger.warning("%s failed to close in __del__", self._log_prefix)
def __enter__(self):
return self
diff --git a/ydb/_topic_writer/topic_writer.py b/ydb/_topic_writer/topic_writer.py
index 18e790fe..f73d3830 100644
--- a/ydb/_topic_writer/topic_writer.py
+++ b/ydb/_topic_writer/topic_writer.py
@@ -47,8 +47,11 @@ class PublicWriterSettings:
# Backpressure is enabled when at least one of the limits above is set.
# None = wait indefinitely for buffer space; positive value = raise TopicWriterBufferFullError on timeout.
buffer_wait_timeout_sec: Optional[float] = None
+ writer_name: Optional[str] = None
def __post_init__(self):
+ if self.writer_name is not None and not isinstance(self.writer_name, str):
+ raise TypeError("Unsupported type for writer_name field: '%s'" % type(self.writer_name))
if self.producer_id is None:
self.producer_id = uuid.uuid4().hex
if self.max_buffer_size_bytes is not None and self.max_buffer_size_bytes <= 0:
@@ -152,6 +155,8 @@ def _create_message(data: Message) -> "PublicMessage":
class InternalMessage(StreamWriteMessage.WriteRequest.MessageData, IToProto):
codec: PublicCodec
+ _accepted_at: Optional[typing.Tuple[Any, float]]
+ _first_sent_at: Optional[typing.Tuple[Any, float]]
def __init__(self, mess: PublicMessage):
metadata_items: Dict[str, bytes] = {}
@@ -178,6 +183,8 @@ def __init__(self, mess: PublicMessage):
partitioning=None,
)
self.codec = PublicCodec(PublicCodec.RAW)
+ self._accepted_at = None
+ self._first_sent_at = None
def _get_bytes(self, obj: Optional[PublicMessage.SimpleSourceType]) -> bytes:
if obj is None:
diff --git a/ydb/_topic_writer/topic_writer_asyncio.py b/ydb/_topic_writer/topic_writer_asyncio.py
index 81b9e719..83490c61 100644
--- a/ydb/_topic_writer/topic_writer_asyncio.py
+++ b/ydb/_topic_writer/topic_writer_asyncio.py
@@ -33,6 +33,7 @@
from .._utilities import AtomicCounter
from .._errors import check_retriable_error
from ..retries import RetrySettings
+from ..observability.metrics import TopicWriterMetrics, create_topic_writer_metrics
from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec
from .._grpc.grpcwrapper.ydb_topic import (
UpdateTokenRequest,
@@ -56,6 +57,20 @@
logger = logging.getLogger(__name__)
+_WRITER_CODEC_NAMES = {
+ PublicCodec.UNSPECIFIED: "unspecified",
+ PublicCodec.RAW: "raw",
+ PublicCodec.GZIP: "gzip",
+ PublicCodec.LZOP: "lzop",
+ PublicCodec.ZSTD: "zstd",
+}
+
+
+def _writer_codec_name(codec: Optional[int]) -> str:
+ if codec is None:
+ return "unspecified"
+ return _WRITER_CODEC_NAMES.get(codec, str(codec))
+
class WriterAsyncIO:
_loop: asyncio.AbstractEventLoop
@@ -256,6 +271,7 @@ class WriterAsyncIOReconnector:
_init_message: StreamWriteMessage.InitRequest
_stream_connected: asyncio.Event
_settings: WriterSettings
+ _metrics: TopicWriterMetrics
_codec: Optional[PublicCodec]
_codec_functions: Dict[PublicCodec, Callable[[bytes], bytes]]
_encode_executor: Optional[concurrent.futures.Executor]
@@ -296,6 +312,12 @@ def __init__(
self._init_info = None
self._stream_connected = asyncio.Event()
self._settings = settings
+ self._metrics = create_topic_writer_metrics(
+ driver,
+ settings.topic,
+ settings.writer_name,
+ buffer_limit_bytes=settings.max_buffer_size_bytes,
+ )
self._tx = tx
self._codec_functions: Dict[PublicCodec, Callable[[bytes], bytes]] = {
@@ -343,23 +365,25 @@ async def close(self, flush: bool):
return
self._closed = True
logger.debug("Close writer reconnector id=%s", self._id)
+ try:
+ if flush:
+ await self.flush()
- if flush:
- await self.flush()
-
- self._stop(TopicWriterStopped())
+ self._stop(TopicWriterStopped())
- for task in self._background_tasks:
- task.cancel()
- await asyncio.wait(self._background_tasks)
+ for task in self._background_tasks:
+ task.cancel()
+ await asyncio.wait(self._background_tasks)
- # if work was stopped before close by error - raise the error
- try:
- self._check_stop()
- except TopicWriterStopped:
- pass
+ # if work was stopped before close by error - raise the error
+ try:
+ self._check_stop()
+ except TopicWriterStopped:
+ pass
- logger.debug("Writer reconnector id=%s was closed", self._id)
+ logger.debug("Writer reconnector id=%s was closed", self._id)
+ finally:
+ self._metrics.close()
async def wait_init(self) -> PublicWriterInitInfo:
while True:
@@ -393,9 +417,18 @@ async def write_with_ack_future(self, messages: List[PublicMessage]) -> List[asy
if self._backpressure_enabled:
await self._acquire_buffer_space(internal_messages)
+ accepted_at = self._metrics.capture_message_accepted_at()
+ for message in internal_messages:
+ message._accepted_at = accepted_at
+
self._messages_future.extend(messages_future)
+ self._metrics.add_pending_messages(accepted_at, len(internal_messages))
+
+ self._metrics.record_sending_messages(len(internal_messages))
+ self._metrics.record_sending_bytes(sum(message.uncompressed_size for message in internal_messages))
if self._codec is not None and self._codec == PublicCodec.RAW:
+ self._record_message_processing(internal_messages, self._codec)
self._add_messages_to_send_queue(internal_messages)
else:
self._messages_for_encode.put_nowait(internal_messages)
@@ -413,6 +446,8 @@ async def _acquire_buffer_space(self, internal_messages: List[InternalMessage])
max_msgs = self._settings.max_buffer_messages
timeout_sec = self._settings.buffer_wait_timeout_sec
deadline = self._loop.time() + timeout_sec if timeout_sec is not None else None
+ wait_started = False
+ wait_started_at: Optional[typing.Tuple[typing.Any, float]] = None
while True:
self._buffer_updated.clear()
@@ -421,6 +456,9 @@ async def _acquire_buffer_space(self, internal_messages: List[InternalMessage])
):
break
self._check_stop()
+ if not wait_started:
+ wait_started = True
+ wait_started_at = self._metrics.capture_buffer_wait_started_at()
if deadline is not None:
assert timeout_sec is not None
remaining = deadline - self._loop.time()
@@ -441,12 +479,29 @@ async def _acquire_buffer_space(self, internal_messages: List[InternalMessage])
new_bytes = sum(internal_message_size_bytes(m) for m in internal_messages)
self._buffer_bytes += new_bytes
self._buffer_messages += len(internal_messages)
+ self._metrics.set_buffer_used_bytes(self._buffer_bytes)
+ if wait_started:
+ self._metrics.record_buffer_wait_duration(wait_started_at)
def _add_messages_to_send_queue(self, internal_messages: List[InternalMessage]):
self._messages.extend(internal_messages)
for m in internal_messages:
self._new_messages.put_nowait(m)
+ def _record_message_processing(
+ self,
+ messages: List[InternalMessage],
+ codec: Optional[PublicCodec],
+ error: Optional[BaseException] = None,
+ ) -> None:
+ codec_name: Optional[str] = None
+ for message in messages:
+ if message._accepted_at is None:
+ continue
+ if codec_name is None:
+ codec_name = _writer_codec_name(codec)
+ self._metrics.record_message_processing_duration(message._accepted_at, codec_name, error)
+
def _prepare_internal_messages(self, messages: List[PublicMessage]) -> List[InternalMessage]:
if self._settings.auto_created_at:
now = datetime.datetime.now(datetime.timezone.utc)
@@ -538,7 +593,9 @@ async def _connection_loop(self):
err = issues.ConnectionLost("gRPC stream cancelled")
err_info = check_retriable_error(err, retry_settings, attempt)
- if not err_info.is_retriable or self._tx is not None: # no retries in tx writer
+ should_retry = err_info.is_retriable and self._tx is None
+ self._metrics.record_session_error(err, should_retry)
+ if not should_retry: # no retries in tx writer
logger.debug("writer reconnector %s stop connection loop due to %s", self._id, err)
self._stop(err)
return
@@ -551,6 +608,7 @@ async def _connection_loop(self):
await asyncio.sleep(err_info.sleep_timeout_seconds)
except Exception as err:
+ self._metrics.record_session_error(err, False)
self._stop(err)
return
finally:
@@ -574,8 +632,15 @@ async def _encode_loop(self):
len(messages),
)
- batch_codec = await self._codec_selector(messages)
- await self._encode_data_inplace(batch_codec, messages)
+ batch_codec = self._codec
+ try:
+ batch_codec = await self._codec_selector(messages)
+ await self._encode_data_inplace(batch_codec, messages)
+ except asyncio.CancelledError:
+ raise
+ except BaseException as err:
+ self._record_message_processing(messages, batch_codec, err)
+ raise
logger.debug(
"writer reconnector %s encoded %s messages",
@@ -583,6 +648,7 @@ async def _encode_loop(self):
len(messages),
)
+ self._record_message_processing(messages, batch_codec)
self._add_messages_to_send_queue(messages)
except BaseException as err:
self._stop(err)
@@ -593,11 +659,12 @@ async def _encode_data_inplace(self, codec: PublicCodec, messages: List[Internal
eventloop = asyncio.get_running_loop()
encode_waiters = []
- encoder_function = self._codec_functions[codec]
-
for message in messages:
encoded_data_futures = eventloop.run_in_executor(
- self._encode_executor, encoder_function, message.get_data_bytes()
+ self._encode_executor,
+ self._encode_with_metrics,
+ codec,
+ message.get_data_bytes(),
)
encode_waiters.append(encoded_data_futures)
@@ -608,6 +675,20 @@ async def _encode_data_inplace(self, codec: PublicCodec, messages: List[Internal
message.codec = codec
message.data = data
+ def _encode_with_metrics(self, codec: PublicCodec, data: bytes) -> bytes:
+ encoder = self._codec_functions[codec]
+ if codec == PublicCodec.RAW:
+ return encoder(data)
+
+ started_at = self._metrics.capture_compression_started_at()
+ try:
+ encoded = encoder(data)
+ except BaseException as err:
+ self._metrics.record_compression_duration(started_at, _writer_codec_name(codec), err)
+ raise
+ self._metrics.record_compression_duration(started_at, _writer_codec_name(codec), None)
+ return encoded
+
async def _codec_selector(self, messages: List[InternalMessage]) -> PublicCodec:
if self._codec is not None:
return self._codec
@@ -660,10 +741,8 @@ async def _codec_selector_by_check_compress(self, messages: List[InternalMessage
def get_compressed_size(codec) -> int:
s = 0
- f = self._codec_functions[codec]
-
for m in test_messages:
- encoded = f(m.get_data_bytes())
+ encoded = self._encode_with_metrics(codec, m.get_data_bytes())
s += len(encoded)
return s
@@ -704,6 +783,7 @@ def _handle_receive_ack(self, ack):
if self._backpressure_enabled:
self._buffer_bytes = max(0, self._buffer_bytes - internal_message_size_bytes(current_message))
self._buffer_messages = max(0, self._buffer_messages - 1)
+ self._metrics.set_buffer_used_bytes(self._buffer_bytes)
self._buffer_updated.set()
write_ack_msg = StreamWriteMessage.WriteResponse.WriteAck
status = ack.message_write_status
@@ -715,8 +795,19 @@ def _handle_receive_ack(self, ack):
result = PublicWriteResult.WrittenInTx()
else:
raise TopicWriterError("internal error - receive unexpected ack message.")
+ self._metrics.record_message_ack_duration(current_message._first_sent_at)
+ self._metrics.complete_pending_message()
+ self._metrics.record_written_messages(1)
message_future.set_result(result)
+ def _capture_first_send(self, messages: List[InternalMessage]) -> None:
+ unsent_messages = [message for message in messages if message._first_sent_at is None]
+ if not unsent_messages:
+ return
+ first_sent_at = self._metrics.capture_message_first_sent_at()
+ for message in unsent_messages:
+ message._first_sent_at = first_sent_at
+
async def _send_loop(self, writer: "WriterAsyncIOStream"):
try:
logger.debug("writer reconnector %s send loop start", self._id)
@@ -724,6 +815,7 @@ async def _send_loop(self, writer: "WriterAsyncIOStream"):
last_seq_no = 0
if messages:
+ self._capture_first_send(messages)
writer.write(messages)
last_seq_no = messages[-1].seq_no
logger.debug(
@@ -745,6 +837,7 @@ async def _send_loop(self, writer: "WriterAsyncIOStream"):
if next_msg.seq_no > last_seq_no:
batch.append(next_msg)
+ self._capture_first_send(batch)
writer.write(batch)
last_seq_no = batch[-1].seq_no
logger.debug(
@@ -775,6 +868,13 @@ def _stop(self, reason: BaseException):
f.set_exception(reason)
f.exception() # mark as retrieved so asyncio does not log "Future exception was never retrieved"
+ self._metrics.clear_pending_messages()
+
+ if self._backpressure_enabled and (self._buffer_bytes or self._buffer_messages):
+ self._buffer_bytes = 0
+ self._buffer_messages = 0
+ self._metrics.set_buffer_used_bytes(0)
+
self._buffer_updated.set() # wake any tasks blocked in _acquire_buffer_space
self._state_changed.set()
logger.info("Stop topic writer %s: %s" % (self._id, reason))
diff --git a/ydb/_topic_writer/topic_writer_asyncio_test.py b/ydb/_topic_writer/topic_writer_asyncio_test.py
index af741660..790c4deb 100644
--- a/ydb/_topic_writer/topic_writer_asyncio_test.py
+++ b/ydb/_topic_writer/topic_writer_asyncio_test.py
@@ -36,6 +36,7 @@
PublicWriteResult,
TopicWriterError,
TopicWriterBufferFullError,
+ internal_message_size_bytes,
)
from .._grpc.grpcwrapper.ydb_topic_public_types import PublicCodec
from .._topic_common.test_helpers import StreamMock, wait_for_fast
@@ -532,8 +533,11 @@ async def test_stop_on_unexpected_exception(self, reconnector: WriterAsyncIOReco
class TestException(Exception):
pass
+ metrics = mock.Mock()
+ reconnector._metrics = metrics
+ test_error = TestException()
stream_writer = get_stream_writer()
- stream_writer.from_server.put_nowait(TestException())
+ stream_writer.from_server.put_nowait(test_error)
message = PublicMessage(
data="123",
@@ -551,6 +555,80 @@ async def wait_stop():
with pytest.raises(TestException):
await reconnector.close(flush=False)
+ metrics.record_session_error.assert_called_once_with(test_error, False)
+ metrics.close.assert_called_once_with()
+
+ async def test_non_retriable_session_error_records_stop(
+ self, reconnector: WriterAsyncIOReconnector, get_stream_writer
+ ):
+ metrics = mock.Mock()
+ reconnector._metrics = metrics
+ test_error = issues.BadRequest("stop")
+ stream_writer = get_stream_writer()
+ stream_writer.from_server.put_nowait(test_error)
+
+ assert await asyncio.wait_for(reconnector.wait_stop(), 1) is test_error
+ metrics.record_session_error.assert_called_once_with(test_error, False)
+
+ with pytest.raises(issues.BadRequest):
+ await reconnector.close(flush=False)
+
+ async def test_retriable_session_error_records_stop_for_transaction_writer(
+ self, default_driver, default_settings, get_stream_writer
+ ):
+ tx = mock.Mock()
+ tx._tx_identity.return_value = FAKE_TRANSACTION_IDENTITY
+ reconnector = WriterAsyncIOReconnector(default_driver, default_settings, tx=tx)
+ metrics = mock.Mock()
+ reconnector._metrics = metrics
+ test_error = issues.Overloaded("transaction writer cannot retry")
+ stream_writer = get_stream_writer()
+ stream_writer.from_server.put_nowait(test_error)
+
+ assert await asyncio.wait_for(reconnector.wait_stop(), 1) is test_error
+ metrics.record_session_error.assert_called_once_with(test_error, False)
+
+ with pytest.raises(issues.Overloaded):
+ await reconnector.close(flush=False)
+
+ async def test_normal_close_does_not_record_session_error(self, reconnector: WriterAsyncIOReconnector):
+ metrics = mock.Mock()
+ reconnector._metrics = metrics
+ await reconnector.wait_init()
+
+ await reconnector.close(flush=False)
+
+ metrics.record_session_error.assert_not_called()
+
+ async def test_encoding_failure_does_not_record_session_error(self, default_driver, default_settings):
+ test_error = ValueError("encode failure")
+
+ def fail_encoding(data):
+ raise test_error
+
+ settings = copy.deepcopy(default_settings)
+ settings.codec = PublicCodec.GZIP
+ settings.encoders = {PublicCodec.GZIP: fail_encoding}
+ settings.max_buffer_messages = 1
+ reconnector = WriterAsyncIOReconnector(default_driver, settings)
+ metrics = mock.Mock()
+ metrics.capture_compression_started_at.return_value = 30.0
+ reconnector._metrics = metrics
+
+ future = (await reconnector.write_with_ack_future([PublicMessage(data=b"data", seqno=1)]))[0]
+ with pytest.raises(ValueError, match="encode failure"):
+ await future
+
+ metrics.record_session_error.assert_not_called()
+ metrics.record_message_ack_duration.assert_not_called()
+ metrics.clear_pending_messages.assert_called_once_with()
+ metrics.capture_compression_started_at.assert_called_once_with()
+ metrics.record_compression_duration.assert_called_once_with(30.0, "gzip", test_error)
+ assert metrics.set_buffer_used_bytes.call_count == 2
+ assert metrics.set_buffer_used_bytes.call_args_list[-1] == mock.call(0)
+ with pytest.raises(ValueError, match="encode failure"):
+ await reconnector.close(flush=False)
+ metrics.close.assert_called_once_with()
async def test_wait_init(self, default_driver, default_settings, get_stream_writer):
init_seqno = 100
@@ -586,10 +664,139 @@ async def test_write_message(self, reconnector: WriterAsyncIOReconnector, get_st
await reconnector.close(flush=False)
- async def test_buffer_full_timeout_raises(self, default_driver, get_stream_writer):
+ async def test_message_metrics_count_acceptance_and_server_acks_once(
+ self,
+ default_driver,
+ get_stream_writer,
+ default_write_statistic,
+ monkeypatch,
+ ):
+ metrics = mock.Mock()
+ metrics.capture_compression_started_at.return_value = 30.0
+ metrics.capture_message_accepted_at.return_value = 20.0
+ metrics.capture_message_first_sent_at.return_value = 40.0
+ metrics_factory = mock.Mock(return_value=metrics)
+ monkeypatch.setattr(
+ "ydb._topic_writer.topic_writer_asyncio.create_topic_writer_metrics",
+ metrics_factory,
+ )
+ settings = WriterSettings(
+ PublicWriterSettings(
+ topic="/local/topic",
+ producer_id="test-producer",
+ auto_seqno=False,
+ auto_created_at=False,
+ codec=PublicCodec.GZIP,
+ max_buffer_messages=3,
+ buffer_wait_timeout_sec=0,
+ writer_name="payments-writer",
+ )
+ )
+ reconnector = WriterAsyncIOReconnector(default_driver, settings)
+ stream_writer = get_stream_writer()
+ retry_error = issues.Overloaded("retry")
+ try:
+ futures = await reconnector.write_with_ack_future(
+ [
+ PublicMessage(data=b"one", seqno=1, metadata_items={"ignored": b"x" * 50}),
+ PublicMessage(data="два", seqno=2),
+ PublicMessage(data=b"tx", seqno=3),
+ ]
+ )
+
+ metrics_factory.assert_called_once_with(
+ default_driver,
+ "/local/topic",
+ "payments-writer",
+ buffer_limit_bytes=None,
+ )
+ metrics.record_sending_messages.assert_called_once_with(3)
+ metrics.record_sending_bytes.assert_called_once_with(11)
+ metrics.add_pending_messages.assert_called_once_with(20.0, 3)
+ metrics.record_written_messages.assert_not_called()
+
+ sent_messages = await stream_writer.from_client.get()
+ buffer_sizes = [internal_message_size_bytes(message) for message in sent_messages]
+ metrics.set_buffer_used_bytes.assert_called_once_with(sum(buffer_sizes))
+ assert metrics.record_compression_duration.call_args_list == [mock.call(30.0, "gzip", None)] * 3
+ metrics.capture_message_accepted_at.assert_called_once_with()
+ metrics.capture_message_first_sent_at.assert_called_once_with()
+ assert {message._first_sent_at for message in sent_messages} == {40.0}
+ assert metrics.record_message_processing_duration.call_args_list == [mock.call(20.0, "gzip", None)] * 3
+ stream_writer.from_server.put_nowait(retry_error)
+ reconnected_writer = get_stream_writer()
+ assert await reconnected_writer.from_client.get() == sent_messages
+ metrics.record_session_error.assert_called_once_with(retry_error, True)
+ metrics.record_sending_messages.assert_called_once_with(3)
+ metrics.record_sending_bytes.assert_called_once_with(11)
+ metrics.record_written_messages.assert_not_called()
+ metrics.set_buffer_used_bytes.assert_called_once_with(sum(buffer_sizes))
+ assert metrics.record_compression_duration.call_count == 3
+ metrics.capture_message_accepted_at.assert_called_once_with()
+ metrics.capture_message_first_sent_at.assert_called_once_with()
+ assert metrics.record_message_processing_duration.call_count == 3
+
+ with pytest.raises(TopicWriterError, match="Empty seqno"):
+ await reconnector.write_with_ack_future([PublicMessage(data=b"invalid")])
+
+ metrics.record_sending_messages.assert_called_once_with(3)
+ metrics.record_sending_bytes.assert_called_once_with(11)
+ metrics.capture_message_accepted_at.assert_called_once_with()
+
+ with pytest.raises(TopicWriterBufferFullError, match="buffer full"):
+ await reconnector.write_with_ack_future([PublicMessage(data=b"full", seqno=4)])
+
+ metrics.record_sending_messages.assert_called_once_with(3)
+ metrics.record_sending_bytes.assert_called_once_with(11)
+ metrics.capture_message_accepted_at.assert_called_once_with()
+ assert metrics.record_message_processing_duration.call_count == 3
+
+ write_ack = StreamWriteMessage.WriteResponse.WriteAck
+ reconnected_writer.from_server.put_nowait(
+ StreamWriteMessage.WriteResponse(
+ partition_id=1,
+ acks=[
+ write_ack(seq_no=1, message_write_status=write_ack.StatusWritten(offset=1)),
+ write_ack(
+ seq_no=2,
+ message_write_status=write_ack.StatusSkipped(
+ reason=write_ack.StatusSkipped.Reason.ALREADY_WRITTEN
+ ),
+ ),
+ write_ack(seq_no=3, message_write_status=write_ack.StatusWrittenInTx()),
+ ],
+ write_statistics=default_write_statistic,
+ )
+ )
+ await asyncio.gather(*futures)
+
+ assert metrics.record_written_messages.call_args_list == [mock.call(1)] * 3
+ assert metrics.record_message_ack_duration.call_args_list == [mock.call(40.0)] * 3
+ assert metrics.complete_pending_message.call_count == 3
+ remaining_bytes = sum(buffer_sizes)
+ expected_buffer_calls = [mock.call(remaining_bytes)]
+ for message_size in buffer_sizes:
+ remaining_bytes -= message_size
+ expected_buffer_calls.append(mock.call(remaining_bytes))
+ assert metrics.set_buffer_used_bytes.call_args_list == expected_buffer_calls
+ finally:
+ await reconnector.close(flush=False)
+ metrics.record_session_error.assert_called_once_with(retry_error, True)
+ metrics.clear_pending_messages.assert_called_once_with()
+ metrics.close.assert_called_once_with()
+
+ async def test_buffer_full_timeout_raises(self, default_driver, get_stream_writer, monkeypatch):
# Soft limit: blocking starts when buffer >= limit.
# First message is 10 bytes data + 64 overhead = 74 bytes; set limit=74 so the
# second write finds buffer already at the limit and must wait.
+ metrics = mock.Mock()
+ metrics.capture_buffer_wait_started_at.return_value = 10.0
+ metrics.capture_message_accepted_at.return_value = 20.0
+ metrics_factory = mock.Mock(return_value=metrics)
+ monkeypatch.setattr(
+ "ydb._topic_writer.topic_writer_asyncio.create_topic_writer_metrics",
+ metrics_factory,
+ )
settings = WriterSettings(
PublicWriterSettings(
topic="/local/topic",
@@ -602,20 +809,43 @@ async def test_buffer_full_timeout_raises(self, default_driver, get_stream_write
)
)
reconnector = WriterAsyncIOReconnector(default_driver, settings)
+ metrics_factory.assert_called_once_with(
+ default_driver,
+ "/local/topic",
+ None,
+ buffer_limit_bytes=74,
+ )
stream_writer = get_stream_writer()
await reconnector.write_with_ack_future([PublicMessage(data=b"x" * 10, seqno=1)])
await stream_writer.from_client.get()
+ metrics.set_buffer_used_bytes.assert_called_once_with(74)
+ metrics.capture_buffer_wait_started_at.assert_not_called()
+ metrics.capture_message_accepted_at.assert_called_once_with()
+ metrics.record_message_processing_duration.assert_called_once_with(20.0, "raw", None)
# buffer == limit (74) → second write blocks and times out
with pytest.raises(TopicWriterBufferFullError, match="buffer full"):
await reconnector.write_with_ack_future([PublicMessage(data=b"y" * 10, seqno=2)])
+ metrics.set_buffer_used_bytes.assert_called_once_with(74)
+ metrics.capture_buffer_wait_started_at.assert_called_once_with()
+ metrics.record_buffer_wait_duration.assert_not_called()
+ metrics.capture_message_accepted_at.assert_called_once_with()
+ metrics.record_message_processing_duration.assert_called_once_with(20.0, "raw", None)
await reconnector.close(flush=False)
+ metrics.close.assert_called_once_with()
- async def test_buffer_freed_by_ack_allows_next_write(self, default_driver, get_stream_writer):
+ async def test_buffer_freed_by_ack_allows_next_write(self, default_driver, get_stream_writer, monkeypatch):
# limit=74 matches one message (10 data + 64 overhead); second write blocks
# until the first is acked and buffer drops to 0 < 74.
+ metrics = mock.Mock()
+ metrics.capture_buffer_wait_started_at.return_value = 10.0
+ metrics.capture_message_accepted_at.return_value = 20.0
+ monkeypatch.setattr(
+ "ydb._topic_writer.topic_writer_asyncio.create_topic_writer_metrics",
+ mock.Mock(return_value=metrics),
+ )
settings = WriterSettings(
PublicWriterSettings(
topic="/local/topic",
@@ -632,23 +862,59 @@ async def test_buffer_freed_by_ack_allows_next_write(self, default_driver, get_s
await reconnector.write_with_ack_future([PublicMessage(data=b"x" * 10, seqno=1)])
await stream_writer.from_client.get()
+ metrics.capture_buffer_wait_started_at.assert_not_called()
+ metrics.capture_message_accepted_at.assert_called_once_with()
+ metrics.record_message_processing_duration.assert_called_once_with(20.0, "raw", None)
- # Ack the first message to free buffer space
- stream_writer.from_server.put_nowait(self.make_default_ack_message(seq_no=1))
+ second_write = asyncio.create_task(
+ reconnector.write_with_ack_future(
+ [
+ PublicMessage(data=b"y" * 10, seqno=2),
+ PublicMessage(data=b"z" * 10, seqno=3),
+ ]
+ )
+ )
+ await asyncio.sleep(0)
+ assert not second_write.done()
+ metrics.capture_buffer_wait_started_at.assert_called_once_with()
+ metrics.record_buffer_wait_duration.assert_not_called()
+ metrics.capture_message_accepted_at.assert_called_once_with()
- # Second write must succeed once buffer is freed
- await reconnector.write_with_ack_future([PublicMessage(data=b"y" * 10, seqno=2)])
+ # Ack the first message to free buffer space; the completed wait is reported once.
+ stream_writer.from_server.put_nowait(self.make_default_ack_message(seq_no=1))
+ await asyncio.wait_for(second_write, 1)
+ metrics.record_buffer_wait_duration.assert_called_once_with(10.0)
+ sent_batch = await asyncio.wait_for(stream_writer.from_client.get(), 1)
+ assert [message.seq_no for message in sent_batch] == [2, 3]
+ assert metrics.capture_message_accepted_at.call_count == 2
+ assert metrics.record_message_processing_duration.call_args_list == [
+ mock.call(20.0, "raw", None),
+ mock.call(20.0, "raw", None),
+ mock.call(20.0, "raw", None),
+ ]
stream_writer.from_server.put_nowait(self.make_default_ack_message(seq_no=2))
+ stream_writer.from_server.put_nowait(self.make_default_ack_message(seq_no=3))
await reconnector.close(flush=True)
- async def test_concurrent_writers_only_one_proceeds_after_ack(self, default_driver, get_stream_writer):
+ async def test_concurrent_writers_only_one_proceeds_after_ack(
+ self,
+ default_driver,
+ get_stream_writer,
+ monkeypatch,
+ ):
# Soft-limit semantics: blocking starts when buffer >= limit.
# limit=74 (one message: 10 data + 64 overhead).
# msg1 fills buffer to 74 >= 74 → tasks 2 and 3 both block.
# Ack msg1 → buffer=0 < 74 → event fires, both tasks wake up.
# First task to run adds 94 bytes (30+64) → buffer=94 >= 74.
# Second task checks again and finds buffer still at limit → stays blocked.
+ metrics = mock.Mock()
+ metrics.capture_buffer_wait_started_at.side_effect = [10.0, 20.0]
+ monkeypatch.setattr(
+ "ydb._topic_writer.topic_writer_asyncio.create_topic_writer_metrics",
+ mock.Mock(return_value=metrics),
+ )
settings = WriterSettings(
PublicWriterSettings(
topic="/local/topic",
@@ -674,6 +940,8 @@ async def test_concurrent_writers_only_one_proceeds_after_ack(self, default_driv
await asyncio.sleep(0)
assert not task2.done()
assert not task3.done()
+ assert metrics.capture_buffer_wait_started_at.call_count == 2
+ metrics.record_buffer_wait_duration.assert_not_called()
# Ack msg1: buffer drops 74 → 0 < 74; one task proceeds and fills buffer again
stream_writer.from_server.put_nowait(self.make_default_ack_message(seq_no=1))
@@ -682,11 +950,15 @@ async def test_concurrent_writers_only_one_proceeds_after_ack(self, default_driv
assert len(done) == 1, "exactly one write should proceed after ack"
assert len(pending) == 1, "other write should still be waiting for buffer space"
assert not next(iter(pending)).done()
+ assert metrics.capture_buffer_wait_started_at.call_count == 2
+ assert metrics.record_buffer_wait_duration.call_count == 1
+ assert metrics.record_buffer_wait_duration.call_args in (mock.call(10.0), mock.call(20.0))
pending_task = next(iter(pending))
pending_task.cancel()
with pytest.raises(asyncio.CancelledError):
await pending_task
+ assert metrics.record_buffer_wait_duration.call_count == 1
await reconnector.close(flush=False)
async def test_buffer_messages_limit_raises_on_timeout(self, default_driver, get_stream_writer):
@@ -842,19 +1114,21 @@ async def test_auto_created_at(self, default_driver, default_settings, get_strea
await reconnector.close(flush=False)
@pytest.mark.parametrize(
- "codec,write_datas,expected_codecs,expected_datas",
+ "codec,write_datas,expected_codecs,expected_datas,expected_compression_calls",
[
(
PublicCodec.RAW,
[b"123"],
[PublicCodec.RAW],
[b"123"],
+ 0,
),
(
PublicCodec.GZIP,
[b"123"],
[PublicCodec.GZIP],
[gzip.compress(b"123", mtime=time_for_mocks)],
+ 1,
),
(
None,
@@ -866,6 +1140,7 @@ async def test_auto_created_at(self, default_driver, default_settings, get_strea
b"789",
b"0" * 1000,
],
+ 2,
),
(
None,
@@ -877,6 +1152,7 @@ async def test_auto_created_at(self, default_driver, default_settings, get_strea
gzip.compress(b"789" * 1000, mtime=time_for_mocks),
gzip.compress(b"0", mtime=time_for_mocks),
],
+ 4,
),
],
)
@@ -889,6 +1165,7 @@ async def test_select_codecs(
codec: typing.Optional[PublicCodec],
expected_codecs: List[PublicCodec],
expected_datas: List[bytes],
+ expected_compression_calls: int,
):
assert len(write_datas) == len(expected_datas)
assert len(expected_codecs) == len(expected_datas)
@@ -897,6 +1174,9 @@ async def test_select_codecs(
settings.codec = codec
settings.auto_seqno = True
reconnector = WriterAsyncIOReconnector(default_driver, settings)
+ metrics = mock.Mock()
+ metrics.capture_compression_started_at.return_value = 30.0
+ reconnector._metrics = metrics
added_messages = asyncio.Queue() # type: asyncio.Queue[List[InternalMessage]]
@@ -920,6 +1200,12 @@ def add_messages(_self, messages: typing.List[InternalMessage]):
else:
assert mess.get_data_bytes() == expected_datas[i]
+ assert metrics.capture_compression_started_at.call_count == expected_compression_calls
+ assert (
+ metrics.record_compression_duration.call_args_list
+ == [mock.call(30.0, "gzip", None)] * expected_compression_calls
+ )
+
await reconnector.close(flush=False)
@pytest.mark.parametrize(
@@ -953,6 +1239,41 @@ async def test_encode_data_inplace(
await reconnector.close(flush=True)
+ async def test_message_processing_metrics_encode_failure(self, default_driver, default_settings, monkeypatch):
+ metrics = mock.Mock()
+ metrics.capture_message_accepted_at.return_value = 10.0
+ monkeypatch.setattr(
+ "ydb._topic_writer.topic_writer_asyncio.create_topic_writer_metrics",
+ mock.Mock(return_value=metrics),
+ )
+ settings = copy.copy(default_settings)
+ settings.codec = PublicCodec.GZIP
+ reconnector = WriterAsyncIOReconnector(default_driver, settings)
+ problem = ValueError("encode failed")
+ monkeypatch.setattr(reconnector, "_encode_data_inplace", mock.AsyncMock(side_effect=problem))
+
+ futures = await reconnector.write_with_ack_future(
+ [
+ PublicMessage(data=b"one", seqno=1),
+ PublicMessage(data=b"two", seqno=2),
+ ]
+ )
+ assert await asyncio.wait_for(reconnector.wait_stop(), 1) is problem
+ metrics.add_pending_messages.assert_called_once_with(10.0, 2)
+ metrics.record_message_ack_duration.assert_not_called()
+ metrics.clear_pending_messages.assert_called_once_with()
+ assert metrics.record_message_processing_duration.call_args_list == [
+ mock.call(10.0, "gzip", problem),
+ mock.call(10.0, "gzip", problem),
+ ]
+ for future in futures:
+ with pytest.raises(ValueError, match="encode failed"):
+ await future
+
+ with pytest.raises(ValueError, match="encode failed"):
+ await reconnector.close(flush=False)
+ metrics.close.assert_called_once_with()
+
async def test_custom_encoder(self, default_driver, default_settings, get_stream_writer):
codec = 10001
@@ -1176,3 +1497,51 @@ async def test_writer_create_failure_does_not_leak_grpc_thread():
finally:
channel.close()
server.stop()
+
+
+def test_writer_settings_reject_non_string_writer_name():
+ with pytest.raises(TypeError, match="writer_name"):
+ PublicWriterSettings(topic="/Root/events", writer_name=42)
+
+
+@pytest.mark.parametrize(
+ "client_class,writer_class_name",
+ [("TopicClient", "TopicWriter"), ("TopicClientAsyncIO", "TopicWriterAsyncIO")],
+)
+def test_topic_clients_forward_writer_name(client_class, writer_class_name):
+ from .. import topic as topic_module
+
+ client = object.__new__(getattr(topic_module, client_class))
+ client._closed = False
+ client._driver = mock.Mock()
+ client._executor = mock.Mock()
+
+ with mock.patch.object(topic_module, writer_class_name, return_value=mock.sentinel.writer) as writer_class:
+ result = client.writer("/Root/events", writer_name="payments-writer")
+
+ assert result is mock.sentinel.writer
+ assert writer_class.call_args.args[1].writer_name == "payments-writer"
+ client._closed = True
+
+
+@pytest.mark.parametrize(
+ "client_class,writer_class_name",
+ [("TopicClient", "TopicTxWriter"), ("TopicClientAsyncIO", "TopicTxWriterAsyncIO")],
+)
+def test_topic_tx_clients_forward_writer_name(client_class, writer_class_name):
+ from .. import topic as topic_module
+
+ client = object.__new__(getattr(topic_module, client_class))
+ client._closed = False
+ client._driver = mock.Mock()
+ client._executor = mock.Mock()
+
+ with mock.patch.object(topic_module, writer_class_name, return_value=mock.sentinel.writer) as writer_class:
+ result = client.tx_writer(mock.sentinel.tx, "/Root/events", writer_name="payments-writer")
+
+ assert result is mock.sentinel.writer
+ settings = writer_class.call_args.kwargs.get("settings")
+ if settings is None:
+ settings = writer_class.call_args.args[2]
+ assert settings.writer_name == "payments-writer"
+ client._closed = True
diff --git a/ydb/observability/__init__.py b/ydb/observability/__init__.py
index fc5836dd..5e8700a5 100644
--- a/ydb/observability/__init__.py
+++ b/ydb/observability/__init__.py
@@ -10,9 +10,9 @@
enable_tracing(OtelTracingProvider()) # or any custom TracingProvider
The SDK itself never imports ``opentelemetry``. Until a backend is enabled,
-operations, query sessions and pools use shared no-op instrumentation without metric
-allocations, timers, locks or state tracking. Enable metrics before creating query
-sessions or pools whose lifecycle should be instrumented.
+operations, query sessions, pools, topic Readers and topic Writers use shared no-op
+instrumentation without metric allocations, timers, locks or state tracking. Enable
+metrics before creating objects whose lifecycle should be instrumented.
"""
from typing import List, Optional
@@ -62,9 +62,9 @@ def enable_metrics(provider: MetricsProvider) -> None:
Calling this a second time replaces the previous backend. To turn metrics off
again call :func:`disable_metrics`. Operation and retry metrics follow the active
- provider immediately. Enable metrics before creating query sessions or pools whose
- lifecycle should be instrumented; their no-op lifecycle trackers are not
- retrofitted later.
+ provider immediately. Enable metrics before creating query sessions, pools, topic
+ Readers or topic Writers whose lifecycle should be instrumented; their no-op
+ lifecycle trackers are not retrofitted later.
"""
_set_metrics_provider(provider)
diff --git a/ydb/observability/metrics.py b/ydb/observability/metrics.py
index 6bc06015..bf4f99b6 100644
--- a/ydb/observability/metrics.py
+++ b/ydb/observability/metrics.py
@@ -2,11 +2,11 @@
The SDK records metrics only after :func:`ydb.observability.enable_metrics` installs a
concrete :class:`MetricsProvider` (OpenTelemetry in :mod:`ydb.opentelemetry`, or any
-custom one). Until then operations, query sessions and pools share no-op
-instrumentation without metric allocations, timers, locks or state tracking. Metrics
-stay independent from tracing, and the SDK never depends on ``opentelemetry`` being
-importable. Enable metrics before creating query sessions or pools whose lifecycle
-should be instrumented.
+custom one). Until then operations, query sessions, pools, topic Readers and topic Writers share
+no-op instrumentation without metric allocations, timers, locks or state tracking.
+Metrics stay independent from tracing, and the SDK never depends on ``opentelemetry``
+being importable. Enable metrics before creating query sessions, pools, topic Readers or topic Writers
+whose lifecycle should be instrumented.
A provider handles three instrument kinds:
@@ -23,8 +23,10 @@
import functools
import inspect
import weakref
-from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Tuple, TypeVar, cast
+from collections import deque
+from typing import Any, Callable, Deque, Dict, Iterable, List, Optional, Protocol, Tuple, TypeVar, cast
+from ydb import issues
from ydb.observability._endpoint import split_endpoint
CLIENT_OPERATION_DURATION = "ydb.client.operation.duration"
@@ -38,8 +40,32 @@
QUERY_SESSION_MIN = "ydb.query.session.min"
RETRY_ATTEMPTS = "ydb.client.retry.attempts"
RETRY_DURATION = "ydb.client.retry.duration"
-
-METRICS_SDK_BUILD_INFO = "ydb-sdk-metrics/0.2.0"
+TOPIC_READER_RECEIVED_MESSAGES = "ydb.topic.reader.received.messages"
+TOPIC_READER_RECEIVED_BYTES = "ydb.topic.reader.received.bytes"
+TOPIC_READER_DELIVERED_MESSAGES = "ydb.topic.reader.delivered.messages"
+TOPIC_READER_COMMIT_QUEUED = "ydb.topic.reader.commit.queued"
+TOPIC_READER_COMMIT_ACKNOWLEDGED = "ydb.topic.reader.commit.acknowledged"
+TOPIC_READER_COMMIT_OFFSET_LAG_MAX = "ydb.topic.reader.commit_offset.lag.max"
+TOPIC_READER_SESSION_ERRORS = "ydb.topic.reader.session.errors"
+TOPIC_READER_COMPRESSION_DURATION = "ydb.topic.reader.compression.duration"
+TOPIC_READER_MESSAGE_PROCESSING_DURATION = "ydb.topic.reader.message.processing.duration"
+TOPIC_READER_CREDIT_BALANCE_BYTES = "ydb.topic.reader.credit_balance_bytes"
+TOPIC_READER_LOCAL_BUFFER_MESSAGES = "ydb.topic.reader.local_buffer.messages"
+TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX = "ydb.topic.reader.local_buffer.message_age.max"
+TOPIC_READER_PARTITION_SESSION_COUNT = "ydb.topic.reader.partition_session.count"
+TOPIC_WRITER_SENDING_MESSAGES = "ydb.topic.writer.sending.messages"
+TOPIC_WRITER_SENDING_BYTES = "ydb.topic.writer.sending.bytes"
+TOPIC_WRITER_WRITTEN_MESSAGES = "ydb.topic.writer.written.messages"
+TOPIC_WRITER_SESSION_ERRORS = "ydb.topic.writer.session.errors"
+TOPIC_WRITER_BUFFER_USED_BYTES = "ydb.topic.writer.buffer.used.bytes"
+TOPIC_WRITER_BUFFER_LIMIT_BYTES = "ydb.topic.writer.buffer.limit.bytes"
+TOPIC_WRITER_BUFFER_WAIT_DURATION = "ydb.topic.writer.buffer.wait.duration"
+TOPIC_WRITER_MESSAGE_PROCESSING_DURATION = "ydb.topic.writer.message.processing.duration"
+TOPIC_WRITER_COMPRESSION_DURATION = "ydb.topic.writer.compression.duration"
+TOPIC_WRITER_MESSAGE_ACK_DURATION = "ydb.topic.writer.message.ack.duration"
+TOPIC_WRITER_SENDING_OLDEST_AGE = "ydb.topic.writer.sending.oldest_age"
+
+METRICS_SDK_BUILD_INFO = "ydb-sdk-metrics/0.25.0"
DURATION_BUCKETS_SECONDS = (
0.001,
@@ -136,6 +162,7 @@ def observe_gauge(self, name, callback):
_provider: MetricsProvider = _NOOP_PROVIDER
_provider_generation = 0
_provider_lock = threading.Lock()
+_TimedMetricStart = Tuple[MetricsProvider, float]
# Trackers created while metrics are active live independently from the current
# provider. This lets a replacement provider observe their current state and prevents
@@ -146,6 +173,8 @@ def observe_gauge(self, name, callback):
_session_max_state: Dict[Tuple, int] = {}
_live_session_metrics: "weakref.WeakSet[SessionMetrics]" = weakref.WeakSet()
_live_pool_metrics: "weakref.WeakSet[QuerySessionPoolMetrics]" = weakref.WeakSet()
+_live_topic_reader_metrics: "weakref.WeakSet[TopicReaderMetrics]" = weakref.WeakSet()
+_live_topic_writer_metrics: "weakref.WeakSet[TopicWriterMetrics]" = weakref.WeakSet()
def is_metrics_enabled() -> bool:
@@ -182,10 +211,125 @@ def _observe_session_min() -> List[Tuple[float, Dict[str, Any]]]:
return [(0, attrs) for _, attrs in _observe_session_max()]
+def _observe_topic_reader_local_buffer_messages() -> List[Tuple[float, Dict[str, Any]]]:
+ with _gauge_lock:
+ values: Dict[Tuple, int] = {}
+ for reader_metrics in _live_topic_reader_metrics:
+ key = tuple(sorted(reader_metrics._base_attributes.items()))
+ values[key] = values.get(key, 0) + reader_metrics._local_buffer_messages
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
+def _observe_topic_reader_credit_balance_bytes() -> List[Tuple[float, Dict[str, Any]]]:
+ with _gauge_lock:
+ values: Dict[Tuple, int] = {}
+ for reader_metrics in _live_topic_reader_metrics:
+ key = tuple(sorted(reader_metrics._base_attributes.items()))
+ values[key] = values.get(key, 0) + reader_metrics._credit_balance_bytes
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
+def _observe_topic_reader_local_buffer_message_age_max() -> List[Tuple[float, Dict[str, Any]]]:
+ now = time.monotonic()
+ with _gauge_lock:
+ values: Dict[Tuple, float] = {}
+ for reader_metrics in _live_topic_reader_metrics:
+ key = tuple(sorted(reader_metrics._base_attributes.items()))
+ oldest_received_at = min(
+ (cohorts[0][0] for cohorts in reader_metrics._local_buffer_received_at.values() if cohorts),
+ default=None,
+ )
+ age = 0.0 if oldest_received_at is None else max(0.0, now - oldest_received_at)
+ values[key] = max(values.get(key, 0.0), age)
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
+def _observe_topic_reader_partition_session_count() -> List[Tuple[float, Dict[str, Any]]]:
+ with _gauge_lock:
+ values: Dict[Tuple, int] = {}
+ for reader_metrics in _live_topic_reader_metrics:
+ for topic, count in reader_metrics._partition_session_counts.items():
+ attributes = dict(reader_metrics._base_attributes)
+ attributes["topic"] = topic
+ key = tuple(sorted(attributes.items()))
+ values[key] = values.get(key, 0) + count
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
+def _observe_topic_reader_commit_offset_lag_max() -> List[Tuple[float, Dict[str, Any]]]:
+ with _gauge_lock:
+ values: Dict[Tuple, int] = {}
+ for reader_metrics in _live_topic_reader_metrics:
+ for topic, requested_offset, acknowledged_offset in reader_metrics._commit_offsets.values():
+ attributes = dict(reader_metrics._base_attributes)
+ attributes["topic"] = topic
+ key = tuple(sorted(attributes.items()))
+ lag = max(0, requested_offset - acknowledged_offset)
+ values[key] = max(values.get(key, 0), lag)
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
+def _observe_topic_writer_buffer_used_bytes() -> List[Tuple[float, Dict[str, Any]]]:
+ with _gauge_lock:
+ values: Dict[Tuple, int] = {}
+ for writer_metrics in _live_topic_writer_metrics:
+ key = tuple(sorted(writer_metrics._base_attributes.items()))
+ values[key] = values.get(key, 0) + writer_metrics._buffer_used_bytes
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
+def _observe_topic_writer_buffer_limit_bytes() -> List[Tuple[float, Dict[str, Any]]]:
+ with _gauge_lock:
+ values: Dict[Tuple, int] = {}
+ unbounded: set = set()
+ for writer_metrics in _live_topic_writer_metrics:
+ key = tuple(sorted(writer_metrics._base_attributes.items()))
+ if writer_metrics._buffer_limit_bytes is None:
+ unbounded.add(key)
+ values.pop(key, None)
+ elif key not in unbounded:
+ values[key] = values.get(key, 0) + writer_metrics._buffer_limit_bytes
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
+def _observe_topic_writer_sending_oldest_age() -> List[Tuple[float, Dict[str, Any]]]:
+ with _gauge_lock:
+ values: Dict[Tuple, float] = {}
+ now: Optional[float] = None
+ for writer_metrics in _live_topic_writer_metrics:
+ key = tuple(sorted(writer_metrics._base_attributes.items()))
+ if writer_metrics._pending_messages_accepted_at:
+ if now is None:
+ now = time.monotonic()
+ age = max(0.0, now - writer_metrics._pending_messages_accepted_at[0][0])
+ else:
+ age = 0.0
+ values[key] = max(values.get(key, 0.0), age)
+ return [(value, dict(attrs)) for attrs, value in values.items()]
+
+
_OBSERVABLE_GAUGES = (
(QUERY_SESSION_COUNT, _observe_session_count),
(QUERY_SESSION_MAX, _observe_session_max),
(QUERY_SESSION_MIN, _observe_session_min),
+ (TOPIC_READER_CREDIT_BALANCE_BYTES, _observe_topic_reader_credit_balance_bytes),
+ (TOPIC_READER_COMMIT_OFFSET_LAG_MAX, _observe_topic_reader_commit_offset_lag_max),
+ (TOPIC_READER_LOCAL_BUFFER_MESSAGES, _observe_topic_reader_local_buffer_messages),
+ (TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX, _observe_topic_reader_local_buffer_message_age_max),
+ (TOPIC_READER_PARTITION_SESSION_COUNT, _observe_topic_reader_partition_session_count),
+ (TOPIC_WRITER_BUFFER_USED_BYTES, _observe_topic_writer_buffer_used_bytes),
+ (TOPIC_WRITER_BUFFER_LIMIT_BYTES, _observe_topic_writer_buffer_limit_bytes),
+ (TOPIC_WRITER_SENDING_OLDEST_AGE, _observe_topic_writer_sending_oldest_age),
+)
+
+_TRANSPORT_STATUS_CODES = frozenset(
+ {
+ issues.StatusCode.CONNECTION_LOST,
+ issues.StatusCode.CONNECTION_FAILURE,
+ issues.StatusCode.DEADLINE_EXCEEDED,
+ issues.StatusCode.CLIENT_INTERNAL_ERROR,
+ issues.StatusCode.UNIMPLEMENTED,
+ }
)
@@ -266,7 +410,7 @@ def query_session_pool_name(
def _metrics_build_info_tokens() -> List[str]:
"""Metrics' contribution to the ``x-ydb-sdk-build-info`` header.
- Returns ``["ydb-sdk-metrics/0.2.0"]`` once a metrics backend is installed,
+ Returns ``["ydb-sdk-metrics/0.25.0"]`` once a metrics backend is installed,
otherwise an empty list. Aggregated with other features by
:func:`ydb.observability.sdk_build_info_tokens`.
"""
@@ -306,6 +450,29 @@ def _response_status_code(exception: BaseException) -> str:
return type(exception).__qualname__
+def _error_type(exception: BaseException) -> str:
+ status = getattr(exception, "status", None)
+ if status in _TRANSPORT_STATUS_CODES or isinstance(exception, issues.ConnectionError):
+ return "transport_error"
+ if isinstance(exception, issues.Error):
+ return "ydb_error"
+ return type(exception).__qualname__
+
+
+def _session_error_attributes(
+ base_attributes: Dict[str, Any], error: BaseException, is_retriable: bool
+) -> Dict[str, Any]:
+ attributes = dict(base_attributes)
+ attributes.update(
+ {
+ "retry_decision": "retry" if is_retriable else "stop",
+ "status_code": _response_status_code(error),
+ "error.type": _error_type(error),
+ }
+ )
+ return attributes
+
+
class MetricsOperation:
"""Metric lifecycle object for one user-visible YDB client operation.
@@ -497,6 +664,624 @@ def __exit__(self, exc_type, exc_val, exc_tb):
_NOOP_CM = _NoopContext()
+class TopicWriterMetrics:
+ """Metric context shared by every stream of one logical topic writer."""
+
+ __slots__ = (
+ "_base_attributes",
+ "_buffer_limit_bytes",
+ "_buffer_used_bytes",
+ "_closed",
+ "_pending_messages_accepted_at",
+ "__weakref__",
+ )
+
+ def __init__(
+ self,
+ driver,
+ topic: str,
+ writer_name: str,
+ buffer_limit_bytes: Optional[int] = None,
+ ) -> None:
+ driver_config = getattr(driver, "_driver_config", None)
+ self._base_attributes = _build_ydb_metrics_attrs(driver_config)
+ self._base_attributes.update(
+ {
+ "topic": topic,
+ "writer.name": writer_name,
+ }
+ )
+ self._buffer_limit_bytes = buffer_limit_bytes
+ self._buffer_used_bytes = 0
+ self._pending_messages_accepted_at: Deque[Tuple[float, int]] = deque()
+ self._closed = False
+ with _gauge_lock:
+ _live_topic_writer_metrics.add(self)
+
+ def record_sending_messages(self, message_count: int) -> None:
+ provider = _provider
+ if provider is not _NOOP_PROVIDER and message_count > 0:
+ provider.add(TOPIC_WRITER_SENDING_MESSAGES, message_count, dict(self._base_attributes))
+
+ def record_sending_bytes(self, bytes_size: int) -> None:
+ provider = _provider
+ if provider is not _NOOP_PROVIDER and bytes_size > 0:
+ provider.add(TOPIC_WRITER_SENDING_BYTES, bytes_size, dict(self._base_attributes))
+
+ def record_written_messages(self, message_count: int) -> None:
+ provider = _provider
+ if provider is not _NOOP_PROVIDER and message_count > 0:
+ provider.add(TOPIC_WRITER_WRITTEN_MESSAGES, message_count, dict(self._base_attributes))
+
+ def record_session_error(self, error: BaseException, is_retriable: bool) -> None:
+ provider = _provider
+ if provider is not _NOOP_PROVIDER:
+ provider.add(
+ TOPIC_WRITER_SESSION_ERRORS,
+ 1,
+ _session_error_attributes(self._base_attributes, error, is_retriable),
+ )
+
+ def capture_buffer_wait_started_at(self) -> Optional[_TimedMetricStart]:
+ return _provider, time.monotonic()
+
+ def record_buffer_wait_duration(self, started_at: Optional[_TimedMetricStart]) -> None:
+ if started_at is not None:
+ provider, timestamp = started_at
+ provider.record(
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ max(0.0, time.monotonic() - timestamp),
+ dict(self._base_attributes),
+ )
+
+ def capture_message_accepted_at(self) -> Optional[_TimedMetricStart]:
+ return _provider, time.monotonic()
+
+ def add_pending_messages(self, accepted_at: Optional[_TimedMetricStart], message_count: int) -> None:
+ if accepted_at is None or message_count <= 0:
+ return
+ _, timestamp = accepted_at
+ with _gauge_lock:
+ if self._closed:
+ return
+ if self._pending_messages_accepted_at and self._pending_messages_accepted_at[-1][0] == timestamp:
+ cohort_accepted_at, cohort_count = self._pending_messages_accepted_at.pop()
+ self._pending_messages_accepted_at.append((cohort_accepted_at, cohort_count + message_count))
+ else:
+ self._pending_messages_accepted_at.append((timestamp, message_count))
+
+ def complete_pending_message(self) -> None:
+ with _gauge_lock:
+ if not self._pending_messages_accepted_at:
+ return
+ accepted_at, message_count = self._pending_messages_accepted_at.popleft()
+ if message_count > 1:
+ self._pending_messages_accepted_at.appendleft((accepted_at, message_count - 1))
+
+ def clear_pending_messages(self) -> None:
+ with _gauge_lock:
+ self._pending_messages_accepted_at.clear()
+
+ def capture_message_first_sent_at(self) -> Optional[_TimedMetricStart]:
+ return _provider, time.monotonic()
+
+ def record_message_ack_duration(self, first_sent_at: Optional[_TimedMetricStart]) -> None:
+ if first_sent_at is not None:
+ provider, timestamp = first_sent_at
+ provider.record(
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ max(0.0, time.monotonic() - timestamp),
+ dict(self._base_attributes),
+ )
+
+ def capture_compression_started_at(self) -> Optional[_TimedMetricStart]:
+ return _provider, time.monotonic()
+
+ def record_compression_duration(
+ self,
+ started_at: Optional[_TimedMetricStart],
+ codec: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ self._record_message_duration(TOPIC_WRITER_COMPRESSION_DURATION, started_at, codec, error)
+
+ def record_message_processing_duration(
+ self,
+ accepted_at: Optional[_TimedMetricStart],
+ codec: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ self._record_message_duration(TOPIC_WRITER_MESSAGE_PROCESSING_DURATION, accepted_at, codec, error)
+
+ def _record_message_duration(
+ self,
+ name: str,
+ started_at: Optional[_TimedMetricStart],
+ codec: str,
+ error: Optional[BaseException],
+ ) -> None:
+ if started_at is None:
+ return
+ provider, timestamp = started_at
+
+ attributes = dict(self._base_attributes)
+ attributes.update(
+ {
+ "codec": codec,
+ "outcome": "error" if error is not None else "ok",
+ }
+ )
+ if error is not None:
+ attributes["error.type"] = type(error).__qualname__
+ provider.record(
+ name,
+ max(0.0, time.monotonic() - timestamp),
+ attributes,
+ )
+
+ def set_buffer_used_bytes(self, bytes_size: int) -> None:
+ with _gauge_lock:
+ if not self._closed:
+ self._buffer_used_bytes = max(0, bytes_size)
+
+ def close(self) -> None:
+ with _gauge_lock:
+ if self._closed:
+ return
+ self._closed = True
+ self._buffer_used_bytes = 0
+ self._pending_messages_accepted_at.clear()
+ _live_topic_writer_metrics.discard(self)
+
+
+class _NoopTopicWriterMetrics(TopicWriterMetrics):
+ __slots__ = ()
+
+ def __init__(self) -> None:
+ pass
+
+ def record_sending_messages(self, message_count: int) -> None:
+ pass
+
+ def record_sending_bytes(self, bytes_size: int) -> None:
+ pass
+
+ def record_written_messages(self, message_count: int) -> None:
+ pass
+
+ def record_session_error(self, error: BaseException, is_retriable: bool) -> None:
+ pass
+
+ def capture_buffer_wait_started_at(self) -> Optional[_TimedMetricStart]:
+ return None
+
+ def record_buffer_wait_duration(self, started_at: Optional[_TimedMetricStart]) -> None:
+ pass
+
+ def capture_message_accepted_at(self) -> Optional[_TimedMetricStart]:
+ return None
+
+ def add_pending_messages(self, accepted_at: Optional[_TimedMetricStart], message_count: int) -> None:
+ pass
+
+ def complete_pending_message(self) -> None:
+ pass
+
+ def clear_pending_messages(self) -> None:
+ pass
+
+ def capture_message_first_sent_at(self) -> Optional[_TimedMetricStart]:
+ return None
+
+ def record_message_ack_duration(self, first_sent_at: Optional[_TimedMetricStart]) -> None:
+ pass
+
+ def capture_compression_started_at(self) -> Optional[_TimedMetricStart]:
+ return None
+
+ def record_compression_duration(
+ self,
+ started_at: Optional[_TimedMetricStart],
+ codec: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ pass
+
+ def record_message_processing_duration(
+ self,
+ accepted_at: Optional[_TimedMetricStart],
+ codec: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ pass
+
+ def set_buffer_used_bytes(self, bytes_size: int) -> None:
+ pass
+
+ def close(self) -> None:
+ pass
+
+
+_NOOP_TOPIC_WRITER_METRICS = _NoopTopicWriterMetrics()
+
+
+def create_topic_writer_metrics(
+ driver,
+ topic: str,
+ writer_name: Optional[str],
+ buffer_limit_bytes: Optional[int] = None,
+) -> TopicWriterMetrics:
+ """Create instrumentation only for explicitly named writers while metrics are active."""
+ if not writer_name or not is_metrics_enabled():
+ return _NOOP_TOPIC_WRITER_METRICS
+
+ return TopicWriterMetrics(driver, topic, writer_name, buffer_limit_bytes)
+
+
+class TopicReaderMetrics:
+ """Metric context shared by every stream of one logical topic reader."""
+
+ __slots__ = (
+ "_base_attributes",
+ "_commit_offsets",
+ "_credit_balance_bytes",
+ "_local_buffer_messages",
+ "_local_buffer_received_at",
+ "_partition_session_counts",
+ "_closed",
+ "__weakref__",
+ )
+
+ def __init__(
+ self,
+ driver,
+ consumer_name: Optional[str],
+ reader_name: str,
+ topics: Iterable[str] = (),
+ ) -> None:
+ driver_config = getattr(driver, "_driver_config", None)
+ self._base_attributes = _build_ydb_metrics_attrs(driver_config)
+ self._base_attributes["reader.name"] = reader_name
+ if consumer_name is not None:
+ self._base_attributes["consumer"] = consumer_name
+ self._commit_offsets: Dict[Tuple[int, int], Tuple[str, int, int]] = {}
+ self._credit_balance_bytes = 0
+ self._local_buffer_messages = 0
+ self._local_buffer_received_at: Dict[Tuple[int, int], Deque[Tuple[float, int]]] = {}
+ self._partition_session_counts: Dict[str, int] = dict.fromkeys(topics, 0)
+ self._closed = False
+ with _gauge_lock:
+ _live_topic_reader_metrics.add(self)
+
+ @staticmethod
+ def _partition_session_key(partition_session: Any) -> Tuple[int, int]:
+ return partition_session.reader_stream_id, partition_session.id
+
+ def capture_received_at(self) -> Optional[float]:
+ return time.monotonic()
+
+ def record_received_messages(self, batches: Iterable[Any], received_at: Optional[float] = None) -> None:
+ messages_by_topic: Dict[str, int] = {}
+ messages_by_partition: Dict[Tuple[int, int], int] = {}
+ for batch in batches:
+ messages = len(batch.messages)
+ if messages > 0:
+ partition_session = batch._partition_session
+ topic = partition_session.topic_path
+ messages_by_topic[topic] = messages_by_topic.get(topic, 0) + messages
+ key = self._partition_session_key(partition_session)
+ messages_by_partition[key] = messages_by_partition.get(key, 0) + messages
+
+ received_messages = sum(messages_by_partition.values())
+ if received_messages > 0:
+ if received_at is None:
+ received_at = time.monotonic()
+ with _gauge_lock:
+ if not self._closed:
+ self._local_buffer_messages += received_messages
+ for key, messages in messages_by_partition.items():
+ cohorts = self._local_buffer_received_at.setdefault(key, deque())
+ if cohorts and cohorts[-1][0] == received_at:
+ _, previous_messages = cohorts.pop()
+ cohorts.append((received_at, previous_messages + messages))
+ else:
+ cohorts.append((received_at, messages))
+
+ provider = _provider
+ if provider is _NOOP_PROVIDER:
+ return
+
+ for topic, messages in messages_by_topic.items():
+ attributes = dict(self._base_attributes)
+ attributes["topic"] = topic
+ provider.add(TOPIC_READER_RECEIVED_MESSAGES, messages, attributes)
+
+ def record_received_bytes(self, bytes_size: int) -> None:
+ provider = _provider
+ if provider is _NOOP_PROVIDER:
+ return
+
+ if bytes_size > 0:
+ provider.add(TOPIC_READER_RECEIVED_BYTES, bytes_size, dict(self._base_attributes))
+
+ def record_delivered_messages(self, message_count: int, topic: str) -> None:
+ provider = _provider
+ if provider is _NOOP_PROVIDER:
+ return
+
+ if message_count > 0:
+ attributes = dict(self._base_attributes)
+ attributes["topic"] = topic
+ provider.add(TOPIC_READER_DELIVERED_MESSAGES, message_count, attributes)
+
+ def record_compression_duration(
+ self,
+ duration: float,
+ codec: str,
+ topic: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ self._record_message_duration(TOPIC_READER_COMPRESSION_DURATION, duration, codec, topic, error)
+
+ def record_message_processing_duration(
+ self,
+ duration: float,
+ codec: str,
+ topic: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ self._record_message_duration(TOPIC_READER_MESSAGE_PROCESSING_DURATION, duration, codec, topic, error)
+
+ def _record_message_duration(
+ self,
+ name: str,
+ duration: float,
+ codec: str,
+ topic: str,
+ error: Optional[BaseException],
+ ) -> None:
+ provider = _provider
+ if provider is _NOOP_PROVIDER:
+ return
+
+ attributes = dict(self._base_attributes)
+ attributes.update(
+ {
+ "topic": topic,
+ "codec": codec,
+ "outcome": "error" if error is not None else "ok",
+ }
+ )
+ if error is not None:
+ attributes["error.type"] = type(error).__qualname__
+ provider.record(name, max(0.0, duration), attributes)
+
+ def record_commit_queued(self, message_count: int, topic: str) -> None:
+ provider = _provider
+ if provider is _NOOP_PROVIDER:
+ return
+
+ if message_count > 0:
+ attributes = dict(self._base_attributes)
+ attributes["topic"] = topic
+ provider.add(TOPIC_READER_COMMIT_QUEUED, message_count, attributes)
+
+ def record_commit_acknowledged(self, message_count: int, topic: str) -> None:
+ provider = _provider
+ if provider is _NOOP_PROVIDER:
+ return
+
+ if message_count > 0:
+ attributes = dict(self._base_attributes)
+ attributes["topic"] = topic
+ provider.add(TOPIC_READER_COMMIT_ACKNOWLEDGED, message_count, attributes)
+
+ def record_commit_offset_queued(self, partition_session: Any, end_offset: int) -> None:
+ key = self._partition_session_key(partition_session)
+ with _gauge_lock:
+ if self._closed or key not in self._commit_offsets:
+ return
+ topic, requested_offset, acknowledged_offset = self._commit_offsets[key]
+ self._commit_offsets[key] = (
+ topic,
+ max(requested_offset, end_offset),
+ acknowledged_offset,
+ )
+
+ def record_commit_offset_acknowledged(self, partition_session: Any, committed_offset: int) -> None:
+ key = self._partition_session_key(partition_session)
+ with _gauge_lock:
+ if self._closed or key not in self._commit_offsets:
+ return
+ topic, requested_offset, acknowledged_offset = self._commit_offsets[key]
+ self._commit_offsets[key] = (
+ topic,
+ requested_offset,
+ max(acknowledged_offset, committed_offset),
+ )
+
+ def record_session_error(self, error: BaseException, is_retriable: bool) -> None:
+ provider = _provider
+ if provider is _NOOP_PROVIDER:
+ return
+ provider.add(
+ TOPIC_READER_SESSION_ERRORS,
+ 1,
+ _session_error_attributes(self._base_attributes, error, is_retriable),
+ )
+
+ def add_credit_balance_bytes(self, bytes_size: int) -> None:
+ with _gauge_lock:
+ if not self._closed:
+ self._credit_balance_bytes += bytes_size
+
+ def reset_credit_balance_bytes(self, bytes_size: int = 0) -> None:
+ with _gauge_lock:
+ self._credit_balance_bytes = bytes_size
+
+ def remove_local_buffer_messages(self, message_count: int, partition_session: Any) -> None:
+ if message_count <= 0:
+ return
+
+ key = self._partition_session_key(partition_session)
+ with _gauge_lock:
+ if self._closed:
+ return
+
+ self._local_buffer_messages = max(0, self._local_buffer_messages - message_count)
+ cohorts = self._local_buffer_received_at.get(key)
+ messages_to_remove = message_count
+ while cohorts and messages_to_remove > 0:
+ received_at, messages = cohorts[0]
+ if messages <= messages_to_remove:
+ cohorts.popleft()
+ messages_to_remove -= messages
+ else:
+ cohorts[0] = (received_at, messages - messages_to_remove)
+ messages_to_remove = 0
+
+ if cohorts is not None and not cohorts:
+ self._local_buffer_received_at.pop(key, None)
+ if self._local_buffer_messages == 0:
+ self._local_buffer_received_at.clear()
+
+ def reset_local_buffer_messages(self) -> None:
+ with _gauge_lock:
+ self._local_buffer_messages = 0
+ self._local_buffer_received_at.clear()
+
+ def add_partition_session(self, topic: str, partition_session: Any = None) -> None:
+ with _gauge_lock:
+ if self._closed:
+ return
+ self._partition_session_counts[topic] = self._partition_session_counts.get(topic, 0) + 1
+ if partition_session is not None:
+ key = self._partition_session_key(partition_session)
+ committed_offset = partition_session.committed_offset
+ self._commit_offsets[key] = (topic, committed_offset, committed_offset)
+
+ def remove_partition_session(self, topic: str, partition_session: Any = None) -> None:
+ with _gauge_lock:
+ if self._closed or topic not in self._partition_session_counts:
+ return
+ self._partition_session_counts[topic] = max(0, self._partition_session_counts[topic] - 1)
+ if partition_session is not None:
+ self._commit_offsets.pop(self._partition_session_key(partition_session), None)
+
+ def reset_partition_sessions(self) -> None:
+ with _gauge_lock:
+ for topic in self._partition_session_counts:
+ self._partition_session_counts[topic] = 0
+ self._commit_offsets.clear()
+
+ def close(self) -> None:
+ with _gauge_lock:
+ if self._closed:
+ return
+ self._closed = True
+ self._commit_offsets.clear()
+ self._credit_balance_bytes = 0
+ self._local_buffer_messages = 0
+ self._local_buffer_received_at.clear()
+ self._partition_session_counts.clear()
+ _live_topic_reader_metrics.discard(self)
+
+
+class _NoopTopicReaderMetrics(TopicReaderMetrics):
+ __slots__ = ()
+
+ def __init__(self) -> None:
+ pass
+
+ def record_received_messages(self, batches: Iterable[Any], received_at: Optional[float] = None) -> None:
+ pass
+
+ def capture_received_at(self) -> Optional[float]:
+ return None
+
+ def record_received_bytes(self, bytes_size: int) -> None:
+ pass
+
+ def record_delivered_messages(self, message_count: int, topic: str) -> None:
+ pass
+
+ def record_compression_duration(
+ self,
+ duration: float,
+ codec: str,
+ topic: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ pass
+
+ def record_message_processing_duration(
+ self,
+ duration: float,
+ codec: str,
+ topic: str,
+ error: Optional[BaseException] = None,
+ ) -> None:
+ pass
+
+ def record_commit_queued(self, message_count: int, topic: str) -> None:
+ pass
+
+ def record_commit_acknowledged(self, message_count: int, topic: str) -> None:
+ pass
+
+ def record_commit_offset_queued(self, partition_session: Any, end_offset: int) -> None:
+ pass
+
+ def record_commit_offset_acknowledged(self, partition_session: Any, committed_offset: int) -> None:
+ pass
+
+ def record_session_error(self, error: BaseException, is_retriable: bool) -> None:
+ pass
+
+ def add_credit_balance_bytes(self, bytes_size: int) -> None:
+ pass
+
+ def reset_credit_balance_bytes(self, bytes_size: int = 0) -> None:
+ pass
+
+ def remove_local_buffer_messages(self, message_count: int, partition_session: Any) -> None:
+ pass
+
+ def reset_local_buffer_messages(self) -> None:
+ pass
+
+ def add_partition_session(self, topic: str, partition_session: Any = None) -> None:
+ pass
+
+ def remove_partition_session(self, topic: str, partition_session: Any = None) -> None:
+ pass
+
+ def reset_partition_sessions(self) -> None:
+ pass
+
+ def close(self) -> None:
+ pass
+
+
+_NOOP_TOPIC_READER_METRICS = _NoopTopicReaderMetrics()
+
+
+def create_topic_reader_metrics(
+ driver,
+ consumer_name: Optional[str],
+ reader_name: Optional[str],
+ topics: Any = (),
+) -> TopicReaderMetrics:
+ """Create instrumentation only for explicitly named readers while metrics are active."""
+ if not reader_name or not is_metrics_enabled():
+ return _NOOP_TOPIC_READER_METRICS
+
+ if isinstance(topics, str) or hasattr(topics, "path"):
+ topics = (topics,)
+ topic_paths = (topic if isinstance(topic, str) else topic.path for topic in topics)
+ return TopicReaderMetrics(driver, consumer_name, reader_name, topic_paths)
+
+
class SessionMetrics:
"""Per-session query-session-count bookkeeping, kept out of the session's own code.
diff --git a/ydb/opentelemetry/metrics_plugin.py b/ydb/opentelemetry/metrics_plugin.py
index 3eed89df..c1b599ed 100644
--- a/ydb/opentelemetry/metrics_plugin.py
+++ b/ydb/opentelemetry/metrics_plugin.py
@@ -30,6 +30,30 @@
QUERY_SESSION_TIMEOUTS,
RETRY_ATTEMPTS,
RETRY_DURATION,
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_COMMIT_QUEUED,
+ TOPIC_READER_COMPRESSION_DURATION,
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_DELIVERED_MESSAGES,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES,
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_READER_PARTITION_SESSION_COUNT,
+ TOPIC_READER_RECEIVED_BYTES,
+ TOPIC_READER_RECEIVED_MESSAGES,
+ TOPIC_READER_SESSION_ERRORS,
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ TOPIC_WRITER_BUFFER_USED_BYTES,
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ TOPIC_WRITER_SENDING_BYTES,
+ TOPIC_WRITER_SENDING_MESSAGES,
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ TOPIC_WRITER_SESSION_ERRORS,
+ TOPIC_WRITER_WRITTEN_MESSAGES,
ATTEMPT_BUCKETS,
DURATION_BUCKETS_SECONDS,
GaugeCallback,
@@ -45,7 +69,49 @@
QUERY_SESSION_COUNT: ("{connection}", "Number of open YDB query sessions."),
QUERY_SESSION_MAX: ("{connection}", "Maximum configured number of YDB query sessions."),
QUERY_SESSION_MIN: ("{connection}", "Minimum configured number of YDB query sessions."),
+ TOPIC_READER_CREDIT_BALANCE_BYTES: (
+ "By",
+ "The protocol credit granted to the server and not yet consumed by read responses.",
+ ),
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX: (
+ "{message}",
+ "The maximum gap between requested and acknowledged commit offsets.",
+ ),
+ TOPIC_READER_LOCAL_BUFFER_MESSAGES: (
+ "{message}",
+ "The number of messages currently buffered by the reader.",
+ ),
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX: (
+ "s",
+ "Age of the oldest message currently buffered by the reader.",
+ ),
+ TOPIC_READER_PARTITION_SESSION_COUNT: (
+ "{session}",
+ "The number of active partition sessions owned by the reader.",
+ ),
+ TOPIC_WRITER_BUFFER_USED_BYTES: (
+ "By",
+ "The occupied budget of the writer buffer limiter.",
+ ),
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES: (
+ "By",
+ "The configured limit of the writer buffer limiter.",
+ ),
+ TOPIC_WRITER_SENDING_OLDEST_AGE: (
+ "s",
+ "The age of the oldest message in the writer's in-flight buffer.",
+ ),
}
+_OBSERVABLE_GAUGE_INSTRUMENTS = frozenset(
+ {
+ TOPIC_READER_CREDIT_BALANCE_BYTES,
+ TOPIC_READER_COMMIT_OFFSET_LAG_MAX,
+ TOPIC_READER_LOCAL_BUFFER_MESSAGE_AGE_MAX,
+ TOPIC_WRITER_BUFFER_LIMIT_BYTES,
+ TOPIC_WRITER_BUFFER_USED_BYTES,
+ TOPIC_WRITER_SENDING_OLDEST_AGE,
+ }
+)
class OtelMetricsProvider:
@@ -92,6 +158,48 @@ def __init__(self, meter: Meter) -> None:
),
bucket_boundaries=ATTEMPT_BUCKETS,
),
+ TOPIC_READER_COMPRESSION_DURATION: _create_histogram(
+ meter,
+ TOPIC_READER_COMPRESSION_DURATION,
+ unit="s",
+ description="The time spent decompressing a message, excluding queueing.",
+ bucket_boundaries=DURATION_BUCKETS_SECONDS,
+ ),
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION: _create_histogram(
+ meter,
+ TOPIC_READER_MESSAGE_PROCESSING_DURATION,
+ unit="s",
+ description="The time from receipt until a message is ready for delivery.",
+ bucket_boundaries=DURATION_BUCKETS_SECONDS,
+ ),
+ TOPIC_WRITER_BUFFER_WAIT_DURATION: _create_histogram(
+ meter,
+ TOPIC_WRITER_BUFFER_WAIT_DURATION,
+ unit="s",
+ description="Time waiting for buffer capacity before a message is accepted.",
+ bucket_boundaries=DURATION_BUCKETS_SECONDS,
+ ),
+ TOPIC_WRITER_COMPRESSION_DURATION: _create_histogram(
+ meter,
+ TOPIC_WRITER_COMPRESSION_DURATION,
+ unit="s",
+ description="The time spent compressing a message, excluding queueing.",
+ bucket_boundaries=DURATION_BUCKETS_SECONDS,
+ ),
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION: _create_histogram(
+ meter,
+ TOPIC_WRITER_MESSAGE_PROCESSING_DURATION,
+ unit="s",
+ description="The time from buffer acceptance until a message is ready for sending.",
+ bucket_boundaries=DURATION_BUCKETS_SECONDS,
+ ),
+ TOPIC_WRITER_MESSAGE_ACK_DURATION: _create_histogram(
+ meter,
+ TOPIC_WRITER_MESSAGE_ACK_DURATION,
+ unit="s",
+ description="Time from first sending a message to the server to its acknowledgement.",
+ bucket_boundaries=DURATION_BUCKETS_SECONDS,
+ ),
}
self._counters: Dict[str, Any] = {
CLIENT_OPERATION_FAILED: meter.create_counter(
@@ -114,6 +222,56 @@ def __init__(self, meter: Meter) -> None:
unit="{request}",
description="Number of requests waiting for a YDB query session.",
),
+ TOPIC_READER_DELIVERED_MESSAGES: meter.create_counter(
+ TOPIC_READER_DELIVERED_MESSAGES,
+ unit="{message}",
+ description="Number of messages delivered by the YDB topic reader to the application.",
+ ),
+ TOPIC_READER_COMMIT_QUEUED: meter.create_counter(
+ TOPIC_READER_COMMIT_QUEUED,
+ unit="{message}",
+ description="The number of messages in commit ranges accepted by the SDK.",
+ ),
+ TOPIC_READER_COMMIT_ACKNOWLEDGED: meter.create_counter(
+ TOPIC_READER_COMMIT_ACKNOWLEDGED,
+ unit="{message}",
+ description="The number of messages acknowledged by regular or transactional commits.",
+ ),
+ TOPIC_READER_RECEIVED_MESSAGES: meter.create_counter(
+ TOPIC_READER_RECEIVED_MESSAGES,
+ unit="{message}",
+ description="The number of messages accepted by the SDK for an active partition session.",
+ ),
+ TOPIC_READER_RECEIVED_BYTES: meter.create_counter(
+ TOPIC_READER_RECEIVED_BYTES,
+ unit="By",
+ description="The protocol bytes_size received in read responses.",
+ ),
+ TOPIC_READER_SESSION_ERRORS: meter.create_counter(
+ TOPIC_READER_SESSION_ERRORS,
+ unit="{error}",
+ description="Number of YDB topic reader session errors.",
+ ),
+ TOPIC_WRITER_SENDING_MESSAGES: meter.create_counter(
+ TOPIC_WRITER_SENDING_MESSAGES,
+ unit="{message}",
+ description="Number of messages accepted by the SDK from the application for sending.",
+ ),
+ TOPIC_WRITER_SENDING_BYTES: meter.create_counter(
+ TOPIC_WRITER_SENDING_BYTES,
+ unit="By",
+ description="Uncompressed message body bytes accepted by the SDK from the application for sending.",
+ ),
+ TOPIC_WRITER_WRITTEN_MESSAGES: meter.create_counter(
+ TOPIC_WRITER_WRITTEN_MESSAGES,
+ unit="{message}",
+ description="Number of messages confirmed by the server, including already written messages.",
+ ),
+ TOPIC_WRITER_SESSION_ERRORS: meter.create_counter(
+ TOPIC_WRITER_SESSION_ERRORS,
+ unit="{error}",
+ description="Number of YDB topic writer session errors by retry decision.",
+ ),
}
def record(self, name: str, value: float, attributes: Optional[Dict[str, Any]] = None) -> None:
@@ -132,7 +290,12 @@ def observe_gauge(self, name: str, callback: GaugeCallback) -> None:
def _otel_callback(_: CallbackOptions):
return [Observation(value, attributes=attrs) for value, attrs in callback()]
- self._meter.create_observable_up_down_counter(
+ create_instrument = (
+ self._meter.create_observable_gauge
+ if name in _OBSERVABLE_GAUGE_INSTRUMENTS
+ else self._meter.create_observable_up_down_counter
+ )
+ create_instrument(
name,
callbacks=[_otel_callback],
unit=unit,
diff --git a/ydb/topic.py b/ydb/topic.py
index e76956ab..09c40f51 100644
--- a/ydb/topic.py
+++ b/ydb/topic.py
@@ -295,6 +295,7 @@ def reader(
auto_partitioning_support: Optional[bool] = True, # Auto partitioning feature flag. Default - True.
event_handler: Optional[TopicReaderEvents.EventHandler] = None,
buffer_release_threshold: float = 0.5,
+ reader_name: Optional[str] = None,
) -> TopicReaderAsyncIO:
logger.debug("Create reader for topic=%s consumer=%s", topic, consumer)
@@ -345,6 +346,7 @@ def writer(
max_buffer_size_bytes: Optional[int] = None,
max_buffer_messages: Optional[int] = None,
buffer_wait_timeout_sec: Optional[float] = None,
+ writer_name: Optional[str] = None,
) -> TopicWriterAsyncIO:
logger.debug("Create writer for topic=%s producer_id=%s", topic, producer_id)
args = locals().copy()
@@ -377,6 +379,7 @@ def tx_writer(
max_buffer_size_bytes: Optional[int] = None,
max_buffer_messages: Optional[int] = None,
buffer_wait_timeout_sec: Optional[float] = None,
+ writer_name: Optional[str] = None,
) -> TopicTxWriterAsyncIO:
logger.debug("Create tx writer for topic=%s tx=%s", topic, tx)
args = locals().copy()
@@ -631,6 +634,7 @@ def reader(
auto_partitioning_support: Optional[bool] = True, # Auto partitioning feature flag. Default - True.
event_handler: Optional[TopicReaderEvents.EventHandler] = None,
buffer_release_threshold: float = 0.5,
+ reader_name: Optional[str] = None,
) -> TopicReader:
logger.debug("Create reader for topic=%s consumer=%s", topic, consumer)
if not decoder_executor:
@@ -679,6 +683,7 @@ def writer(
max_buffer_size_bytes: Optional[int] = None,
max_buffer_messages: Optional[int] = None,
buffer_wait_timeout_sec: Optional[float] = None,
+ writer_name: Optional[str] = None,
) -> TopicWriter:
logger.debug("Create writer for topic=%s producer_id=%s", topic, producer_id)
args = locals().copy()
@@ -712,6 +717,7 @@ def tx_writer(
max_buffer_size_bytes: Optional[int] = None,
max_buffer_messages: Optional[int] = None,
buffer_wait_timeout_sec: Optional[float] = None,
+ writer_name: Optional[str] = None,
) -> TopicWriter:
logger.debug("Create tx writer for topic=%s tx=%s", topic, tx)
args = locals().copy()