From fffce0ac9707342855fb55ad6ff5b7c016c8c7c3 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Wed, 11 Nov 2026 19:14:30 +0300 Subject: [PATCH 1/6] Add opt-in primary Bridge pile preference --- CHANGELOG.md | 2 + docs/driver.rst | 26 +++ tests/aio/test_discovery_detect_local_dc.py | 178 ++++++++++++++++++++ tests/test_discovery_detect_local_dc.py | 162 ++++++++++++++++++ ydb/aio/driver.py | 5 + ydb/aio/pool.py | 37 ++-- ydb/driver.py | 11 ++ ydb/pool.py | 78 +++++++-- ydb/resolver.py | 28 ++- 9 files changed, 494 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a8302ae7..714f276c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Add opt-in PRIMARY/PROMOTED Bridge pile preference with synchronized-pile fallback for sync and async drivers + ## 3.35.0 ## * Include streaming queries in scheme directory listings and path descriptions * Add timeout to the token exchange request so a stalled token endpoint cannot hang credential refresh diff --git a/docs/driver.rst b/docs/driver.rst index d58762400..e94ce1035 100644 --- a/docs/driver.rst +++ b/docs/driver.rst @@ -48,6 +48,9 @@ DriverConfig * - ``discovery_request_timeout`` - ``10`` - Timeout in seconds for each discovery request. + * - ``prefer_primary_pile`` + - ``False`` + - Prefer PRIMARY/PROMOTED Bridge endpoints with synchronized-pile fallback; no effect without Bridge metadata. * - ``table_client_settings`` - ``None`` - :class:`~ydb.TableClientSettings` instance to configure the Table service client. @@ -62,6 +65,29 @@ DriverConfig - gRPC-level compression (e.g. ``grpc.Compression.Gzip``). +Preferring the primary Bridge pile +---------------------------------------- + +Enable this policy explicitly for sync or async drivers: + +.. code-block:: python + + config = ydb.DriverConfig( + endpoint="grpc://discovery.example:2135", + database="/Root/mydb", + prefer_primary_pile=True, + ) + driver = ydb.Driver(config) + +PRIMARY and PROMOTED endpoints take priority over SYNCHRONIZED endpoints. +Other or unknown pile states are last-resort candidates. The policy follows role +changes from discovery without recreating the driver, and takes precedence over +local-DC preferences while Bridge metadata is present, even with ``use_all_nodes=True``. +Without Bridge metadata the existing balancing behavior is preserved. +Requests bound to a session retain their node affinity; the option does not move +an existing session or perform a server-side failover. It is disabled by default. + + Creating a Driver ----------------- diff --git a/tests/aio/test_discovery_detect_local_dc.py b/tests/aio/test_discovery_detect_local_dc.py index ad534291d..a090c6d39 100644 --- a/tests/aio/test_discovery_detect_local_dc.py +++ b/tests/aio/test_discovery_detect_local_dc.py @@ -2,6 +2,7 @@ import pytest from unittest.mock import MagicMock, patch, AsyncMock from ydb import driver, connection +from ydb import _apis, resolver from ydb.aio import pool, _utilities @@ -141,3 +142,180 @@ def mock_init(self, endpoint, driver_config, endpoint_options=None): await discovery.execute_discovery() assert detect_mock.call_count == 0, "detect_local_dc should NOT be called when use_all_nodes=True" + + +class PrimaryConnection: + failed = set() + created = [] + + def __init__(self, endpoint, driver_config, endpoint_options=None): + self.endpoint = endpoint + self.node_id = endpoint_options.node_id + self.callbacks = [] + self.closed = False + self.created.append(self) + + async def connection_ready(self, ready_timeout): + if self.endpoint in self.failed: + raise TimeoutError() + + def add_cleanup_callback(self, callback): + self.callbacks.append(callback) + + async def close(self, grace=30): + self.closed = True + for callback in self.callbacks: + callback(self) + + +def primary_result(states, nodes=((1, "A"), (2, "B"))): + message = _apis.ydb_discovery.ListEndpointsResult(self_location="local") + for node_id, pile in nodes: + message.endpoints.add( + address=f"node{node_id}", + port=2135, + node_id=node_id, + location="local" if pile == "A" else "remote", + bridge_pile_name=pile, + ) + for name, state in states.items(): + message.pile_states.add(pile_name=name, state=state) + response = _apis.ydb_discovery.ListEndpointsResponse() + response.operation.ready = True + response.operation.status = 400000 + response.operation.result.Pack(message) + return resolver.DiscoveryResult.from_response(None, response, use_all_nodes=True) + + +def primary_discovery(monkeypatch, result, enabled=True, use_all_nodes=True, failed=()): + PrimaryConnection.failed, PrimaryConnection.created = set(failed), [] + monkeypatch.setattr(pool, "Connection", PrimaryConnection) + config = driver.DriverConfig("grpc://test:2135", "/local", use_all_nodes=use_all_nodes, prefer_primary_pile=enabled) + discovery = pool.Discovery(pool.ConnectionsCache(use_all_nodes), config) + discovery._resolver.resolve = AsyncMock(return_value=result) + return discovery + + +@pytest.mark.asyncio +async def test_primary_role_change_and_affinity(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1} + assert (await discovery._cache.get(connection.EndpointKey("node2:2135", 2))).node_id == 2 + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {2} + assert len(PrimaryConnection.created) == 2 + assert not any(conn.closed for conn in PrimaryConnection.created) + assert (await discovery._cache.get(connection.EndpointKey("node1:2135", 1))).node_id == 1 + await (await discovery._cache.get()).close() + assert (await discovery._cache.get()).node_id == 1 + + +@pytest.mark.asyncio +async def test_primary_off_and_ordinary_cluster_locality(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), enabled=False) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1, 2} + discovery = primary_discovery(monkeypatch, primary_result({}), use_all_nodes=False) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1} + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}) + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 2 + discovery._resolver.resolve.return_value = primary_result({}) + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 1 + + +@pytest.mark.asyncio +async def test_primary_promoted_and_unknown_states(monkeypatch): + discovery = primary_discovery( + monkeypatch, + primary_result({"A": 1, "B": 2, "C": 3, "D": 99}, ((1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "missing"))), + ) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1, 2} + await (await discovery._cache.get(connection.EndpointKey("node1:2135", 1))).close() + await (await discovery._cache.get(connection.EndpointKey("node2:2135", 2))).close() + assert (await discovery._cache.get()).node_id == 3 + await (await discovery._cache.get()).close() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {4, 5} + + +@pytest.mark.asyncio +async def test_primary_full_cache_admission_and_failed_connection(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert await discovery.execute_discovery() and discovery._cache.size == 9 + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}, old + ((10, "B"),)) + PrimaryConnection.failed = {"node10:2135"} + assert await discovery.execute_discovery() + assert discovery._cache.size == 9 + assert not any(conn.closed for conn in PrimaryConnection.created if conn.node_id != 10) + PrimaryConnection.failed.clear() + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 10 + assert discovery._cache.size == 9 + created_count = len(PrimaryConnection.created) + assert await discovery.execute_discovery() + assert len(PrimaryConnection.created) == created_count + + +@pytest.mark.asyncio +async def test_primary_cache_preserves_synchronized_fallback(monkeypatch): + nodes = tuple((i, "A") for i in range(1, 11)) + ((11, "B"),) + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}, nodes)) + assert await discovery.execute_discovery() and discovery._cache.size == 9 + assert discovery._cache.already_exists("node11:2135") + for conn in discovery._cache.values(): + if conn.node_id != 11: + await conn.close() + assert (await discovery._cache.get()).node_id == 11 + + +@pytest.mark.asyncio +async def test_primary_alias_retirement_keeps_node_affinity(): + cache = pool.ConnectionsCache() + options = connection.EndpointOptions(node_id=1) + old = PrimaryConnection("old:2135", None, options) + new = PrimaryConnection("new:2135", None, options) + cache.add(old) + cache.add(new) + await old.close() + assert await cache.get(connection.EndpointKey(None, 1)) is new + + +@pytest.mark.asyncio +async def test_primary_driver_keyword_and_config_override(monkeypatch): + from ydb import topic + from ydb.aio import driver as aio_driver, table, scheme, coordination + + seen = [] + monkeypatch.setattr(pool.ConnectionPool, "__init__", lambda self, config: seen.append(config)) + for module, name in ( + (topic, "TopicClientAsyncIO"), + (table, "TableClient"), + (scheme, "SchemeClient"), + (coordination, "CoordinationClient"), + ): + monkeypatch.setattr(module, name, MagicMock()) + aio_driver.Driver(endpoint="grpc://test:2135", database="/local", prefer_primary_pile=True) + assert seen[-1].prefer_primary_pile + config = aio_driver.DriverConfig("grpc://test:2135", "/local", prefer_primary_pile=True) + aio_driver.Driver(config) + assert seen[-1].prefer_primary_pile + aio_driver.Driver(config, prefer_primary_pile=False) + assert not seen[-1].prefer_primary_pile + + +@pytest.mark.asyncio +async def test_primary_discovery_cancellation_closes_candidate(monkeypatch): + import asyncio + + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + monkeypatch.setattr(PrimaryConnection, "connection_ready", AsyncMock(side_effect=asyncio.CancelledError)) + with pytest.raises(asyncio.CancelledError): + await discovery.execute_discovery() + assert len(PrimaryConnection.created) == 1 + assert PrimaryConnection.created[0].closed diff --git a/tests/test_discovery_detect_local_dc.py b/tests/test_discovery_detect_local_dc.py index 874ffba70..6cea9eb2a 100644 --- a/tests/test_discovery_detect_local_dc.py +++ b/tests/test_discovery_detect_local_dc.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from unittest.mock import Mock, MagicMock, patch from ydb import driver, pool, _utilities, connection +from ydb import _apis, resolver class MockEndpointInfo: @@ -118,3 +119,164 @@ def test_detect_local_dc_skipped_when_use_all_nodes_true(): discovery.execute_discovery() assert detect_mock.call_count == 0, "detect_local_dc should NOT be called when use_all_nodes=True" + + +class PrimaryConnection: + def __init__(self, endpoint, node_id): + self.endpoint, self.node_id = endpoint, node_id + self.closed = False + self.callbacks = [] + + def add_cleanup_callback(self, callback): + self.callbacks.append(callback) + + def close(self): + self.closed = True + for callback in self.callbacks: + callback(self) + + +def primary_result(states, nodes=((1, "A"), (2, "B"))): + message = _apis.ydb_discovery.ListEndpointsResult(self_location="local") + for node_id, pile in nodes: + message.endpoints.add( + address=f"node{node_id}", + port=2135, + node_id=node_id, + location="local" if pile == "A" else "remote", + bridge_pile_name=pile, + ) + for name, state in states.items(): + message.pile_states.add(pile_name=name, state=state) + response = _apis.ydb_discovery.ListEndpointsResponse() + response.operation.ready = True + response.operation.status = 400000 + response.operation.result.Pack(message) + return resolver.DiscoveryResult.from_response(None, response, use_all_nodes=True) + + +def primary_discovery(monkeypatch, result, enabled=True, use_all_nodes=True, failed=()): + created = [] + + def ready(endpoint, config, timeout, endpoint_options): + if endpoint in failed: + return None + conn = PrimaryConnection(endpoint, endpoint_options.node_id) + created.append(conn) + return conn + + monkeypatch.setattr(connection.Connection, "ready_factory", ready) + config = driver.DriverConfig("grpc://test:2135", "/local", use_all_nodes=use_all_nodes, prefer_primary_pile=enabled) + discovery = pool.Discovery(pool.ConnectionsCache(use_all_nodes), config) + discovery._resolver = MagicMock() + discovery._resolver.context_resolve.return_value.__enter__.return_value = result + return discovery, created + + +def test_primary_role_change_updates_existing_connections_and_affinity(monkeypatch): + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1} + assert discovery._cache.get(connection.EndpointKey("node2:2135", 2)).node_id == 2 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({"A": 3, "B": 1}) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {2} + assert len(created) == 2 and not any(conn.closed for conn in created) + assert discovery._cache.get(connection.EndpointKey("node1:2135", 1)).node_id == 1 + discovery._cache.get().close() + assert discovery._cache.get().node_id == 1 + + +def test_primary_option_is_off_by_default_and_ordinary_clusters_keep_locality(monkeypatch): + assert not driver.DriverConfig("grpc://test:2135").prefer_primary_pile + discovery, _ = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), enabled=False) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1, 2} + discovery, created = primary_discovery(monkeypatch, primary_result({}), use_all_nodes=False) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1} + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({"A": 3, "B": 1}) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 2 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({}) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 1 + assert len(created) == 2 + + +def test_primary_promoted_synchronized_and_unknown_fallback(monkeypatch): + result = primary_result({"A": 1, "B": 2, "C": 3, "D": 99}, ((1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "missing"))) + discovery, _ = primary_discovery(monkeypatch, result) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1, 2} + discovery._cache.get(connection.EndpointKey("node1:2135", 1)).close() + discovery._cache.get(connection.EndpointKey("node2:2135", 2)).close() + assert discovery._cache.get().node_id == 3 + discovery._cache.get().close() + assert {discovery._cache.get().node_id for _ in range(10)} == {4, 5} + + +def test_primary_new_endpoint_enters_full_cache(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert discovery.execute_discovery() and discovery._cache.size == 9 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result( + {"A": 3, "B": 1}, old + ((10, "B"),) + ) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 10 + assert discovery._cache.size == 9 and sum(conn.closed for conn in created) == 1 + assert discovery.execute_discovery() + assert discovery._cache.size == 9 and len(created) == 10 + + +def test_failed_primary_does_not_evict_full_fallback_cache(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery, created = primary_discovery( + monkeypatch, primary_result({"A": 3, "B": 1}, old + ((10, "B"),)), failed={"node10:2135"} + ) + assert discovery.execute_discovery() + assert discovery._cache.size == 9 and not any(conn.closed for conn in created) + assert discovery._cache.get().node_id in range(1, 10) + + +def test_primary_cache_retains_synchronized_fallback(monkeypatch): + nodes = tuple((i, "A") for i in range(1, 11)) + ((11, "B"),) + discovery, _ = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}, nodes)) + assert discovery.execute_discovery() + assert discovery._cache.size == 9 + assert discovery._cache.already_exists("node11:2135") + for conn in discovery._cache.values(): + if conn.node_id != 11: + conn.close() + assert discovery._cache.get().node_id == 11 + + +def test_primary_connection_retirement_preserves_other_address_node_affinity(): + cache = pool.ConnectionsCache() + old, new = PrimaryConnection("old:2135", 1), PrimaryConnection("new:2135", 1) + cache.add(old) + cache.add(new) + old.close() + assert cache.get(connection.EndpointKey(None, 1)) is new + + +def test_primary_driver_keyword_and_config_override(monkeypatch): + from ydb import topic, table, scheme, coordination + + seen = [] + monkeypatch.setattr(pool.ConnectionPool, "__init__", lambda self, config: seen.append(config)) + for module, name in ( + (topic, "TopicClient"), + (table, "TableClient"), + (scheme, "SchemeClient"), + (coordination, "CoordinationClient"), + ): + monkeypatch.setattr(module, name, MagicMock()) + driver.Driver(endpoint="grpc://test:2135", database="/local", prefer_primary_pile=True) + assert seen[-1].prefer_primary_pile + config = driver.DriverConfig("grpc://test:2135", "/local", prefer_primary_pile=True) + driver.Driver(config) + assert seen[-1].prefer_primary_pile + driver.Driver(config, prefer_primary_pile=False) + assert not seen[-1].prefer_primary_pile diff --git a/ydb/aio/driver.py b/ydb/aio/driver.py index 88221e947..e1532366e 100644 --- a/ydb/aio/driver.py +++ b/ydb/aio/driver.py @@ -56,11 +56,16 @@ def __init__( database: Optional[str] = None, root_certificates: Optional[bytes] = None, credentials: Optional["Credentials"] = None, + *, + prefer_primary_pile: Optional[bool] = None, **kwargs: Any, ) -> None: from .. import topic # local import for prevent cycle import error from . import coordination # local import for prevent cycle import error + if prefer_primary_pile is not None: + kwargs["prefer_primary_pile"] = prefer_primary_pile + config = get_config( driver_config, connection_string, diff --git a/ydb/aio/pool.py b/ydb/aio/pool.py index 150274368..c971b26ea 100644 --- a/ydb/aio/pool.py +++ b/ydb/aio/pool.py @@ -3,6 +3,7 @@ import asyncio import logging import random +import grpc from typing import Any, Callable, Optional, Tuple, TYPE_CHECKING, cast from ydb import issues @@ -47,15 +48,7 @@ async def get( # async version with different Connection type if preferred_endpoint is not None and preferred_endpoint.endpoint in self.connections: return self.connections[preferred_endpoint.endpoint] # type: ignore[return-value] - for conn_lst in self.conn_lst_order: - try: - endpoint, connection = conn_lst.popitem(last=False) - conn_lst[endpoint] = connection - return connection # type: ignore[return-value] - except KeyError: - continue - - raise issues.ConnectionLost("Couldn't find valid connection") + return self._get_balanced_connection() # type: ignore[return-value] def add(self, connection: Optional[Connection], preferred: bool = False) -> bool: # type: ignore[override] # async Connection type if connection is None: @@ -81,7 +74,8 @@ def complete_discovery(self, error: Optional[Exception]) -> None: self._fast_fail_event.set() def remove(self, connection: Connection) -> None: # type: ignore[override] # async Connection type - self.connections_by_node_id.pop(connection.node_id, None) + if cast(Any, self.connections_by_node_id.get(connection.node_id)) is connection: + self.connections_by_node_id.pop(connection.node_id, None) self.preferred.pop(connection.endpoint, None) self.connections.pop(connection.endpoint, None) self.outdated.pop(connection.endpoint, None) @@ -147,9 +141,11 @@ async def execute_discovery(self) -> bool: self._cache.make_outdated(cached_endpoint) local_dc = resolve_details.self_location + priorities = resolve_details.bridge_priorities(True) if self._driver_config.prefer_primary_pile else None + self._cache.set_bridge_priorities(priorities) # Detect local DC using TCP latency if enabled and preferred is meaningful - if self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: + if priorities is None and self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: # Use only endpoints that match the SSL requirements for detection ssl_filtered_endpoints = [ endpoint @@ -200,13 +196,24 @@ async def execute_discovery(self) -> bool: endpoint, endpoint_options, ) in resolved_endpoint.endpoints_with_options(): - if self._cache.size >= self._max_size or self._cache.already_exists(endpoint): + if self._cache.already_exists(endpoint) or not self._cache.connection_slot(endpoint, self._max_size)[0]: continue ready_connection = Connection(endpoint, self._driver_config, endpoint_options=endpoint_options) - await ready_connection.connection_ready(ready_timeout=self._ready_timeout) - - self._cache.add(ready_connection, preferred) + try: + await ready_connection.connection_ready(ready_timeout=self._ready_timeout) + except BaseException as error: + await ready_connection.close(grace=0) + if priorities is not None and isinstance(error, (asyncio.TimeoutError, grpc.RpcError)): + continue + raise + allowed, replacement = self._cache.connection_slot(endpoint, self._max_size) + if allowed: + if replacement is not None: + self._cache.make_outdated(replacement) + self._cache.add(ready_connection, preferred) + else: + await ready_connection.close() await self._cache.cleanup_outdated() return self._cache.size > 0 diff --git a/ydb/driver.py b/ydb/driver.py index 988649804..0d6361f93 100644 --- a/ydb/driver.py +++ b/ydb/driver.py @@ -113,6 +113,7 @@ class DriverConfig(object): "compression", "disable_discovery", "detect_local_dc", + "prefer_primary_pile", "_additional_sdk_headers", ) @@ -141,6 +142,7 @@ def __init__( disable_discovery: bool = False, detect_local_dc: bool = False, *, + prefer_primary_pile: bool = False, _additional_sdk_headers: Tuple[str, ...] = (), ) -> None: """ @@ -167,6 +169,9 @@ def __init__( server-provided self_location. **Note**: This option only affects endpoint selection when use_all_nodes=False.\ When use_all_nodes=True (default), all endpoints are used regardless of detected location. :param _additional_sdk_headers: Reserved for SDK integrations (e.g. dbapi, sqlalchemy). Do not use in application code. + :param prefer_primary_pile: Prefer PRIMARY/PROMOTED Bridge endpoints, with synchronized-pile fallback.\ + Disabled by default and has no effect without Bridge metadata. Overrides locality while in Bridge mode;\ + existing sessions retain node affinity. """ self.endpoint = endpoint @@ -196,6 +201,7 @@ def __init__( self.compression = compression self.disable_discovery = disable_discovery self.detect_local_dc = detect_local_dc + self.prefer_primary_pile = prefer_primary_pile self._additional_sdk_headers = _additional_sdk_headers def set_database(self, database: str) -> "DriverConfig": @@ -301,6 +307,8 @@ def __init__( database: Optional[str] = None, root_certificates: Optional[bytes] = None, credentials: Optional["Credentials"] = None, + *, + prefer_primary_pile: Optional[bool] = None, **kwargs: Any, ) -> None: """ @@ -313,6 +321,7 @@ def __init__( :param endpoint: An endpoint specified in the following format: ://: :param database: A database path :param credentials: A credentials. If not specified credentials constructed by default. + :param prefer_primary_pile: Override the DriverConfig PRIMARY-pile preference when specified. """ from . import topic # local import for prevent cycle import error from . import coordination # local import for prevent cycle import error @@ -325,6 +334,8 @@ def __init__( root_certificates, credentials, ) + if prefer_primary_pile is not None: + driver_config.prefer_primary_pile = prefer_primary_pile super(Driver, self).__init__(driver_config) diff --git a/ydb/pool.py b/ydb/pool.py index 2ba1c33ab..50ee37ea9 100644 --- a/ydb/pool.py +++ b/ydb/pool.py @@ -7,7 +7,7 @@ from concurrent import futures import collections import random -from typing import Any, Callable, ContextManager, List, Optional, Set, Tuple, TYPE_CHECKING +from typing import Any, Callable, ContextManager, Dict, List, Optional, Set, Tuple, TYPE_CHECKING from . import connection as connection_impl, issues, resolver, _utilities, tracing from .observability.tracing import SpanName, create_ydb_span @@ -37,6 +37,48 @@ def __init__(self, use_all_nodes: bool = False, tracer: tracing.Tracer = tracing self.use_all_nodes = use_all_nodes self.conn_lst_order = (self.connections,) if self.use_all_nodes else (self.preferred, self.connections) self.fast_fail_subscriptions: Set["futures.Future[None]"] = set() + self._bridge_priorities: Optional[Dict[str, int]] = None + + def set_bridge_priorities(self, priorities: Optional[Dict[str, int]]) -> None: + with self.lock: + self._bridge_priorities = priorities + + def connection_slot(self, endpoint: str, max_size: int) -> Tuple[bool, Optional[Connection]]: + with self.lock: + active = [conn for key, conn in self.connections.items() if key not in self.outdated] + if len(active) < max_size: + return True, None + if self._bridge_priorities is None or not active: + return False, None + priorities = self._bridge_priorities + target = priorities.get(endpoint, 2) + worst = max(active, key=lambda conn: priorities.get(conn.endpoint, 2)) + ranks = [priorities.get(conn.endpoint, 2) for conn in active] + if target < max(ranks): + # Keep one synchronized connection for fallback when the cache has room for both tiers. + if max_size > 1 and target == 0 and max(ranks) == 1 and ranks.count(1) == 1 and 0 in ranks: + return False, None + return True, worst + if max_size > 1 and target == 1 and 1 not in ranks and ranks.count(0) > 1: + return True, next(conn for conn in active if priorities.get(conn.endpoint, 2) == 0) + return False, None + + def _get_balanced_connection(self) -> Connection: + if self._bridge_priorities is not None and self.connections: + priorities = self._bridge_priorities + best = min(priorities.get(endpoint, 2) for endpoint in self.connections) + for endpoint, conn in self.connections.items(): + if priorities.get(endpoint, 2) == best: + self.connections.move_to_end(endpoint) + return conn + for conn_lst in self.conn_lst_order: + try: + endpoint, conn = conn_lst.popitem(last=False) + conn_lst[endpoint] = conn + return conn + except KeyError: + continue + raise issues.ConnectionLost("Couldn't find valid connection") def add(self, connection: Optional[Connection], preferred: bool = False) -> bool: if connection is None: @@ -142,20 +184,14 @@ def get(self, preferred_endpoint: Optional[EndpointKey] = None) -> Connection: if preferred_endpoint is not None and preferred_endpoint.endpoint in self.connections: return self.connections[preferred_endpoint.endpoint] - for conn_lst in self.conn_lst_order: - try: - endpoint, connection = conn_lst.popitem(last=False) - conn_lst[endpoint] = connection - tracing.trace(self.tracer, {"found_in_lists": True}) - return connection - except KeyError: - continue - - raise issues.ConnectionLost("Couldn't find valid connection") + connection = self._get_balanced_connection() + tracing.trace(self.tracer, {"found_in_lists": True}) + return connection def remove(self, connection: Connection) -> None: with self.lock: - self.connections_by_node_id.pop(connection.node_id, None) + if self.connections_by_node_id.get(connection.node_id) is connection: + self.connections_by_node_id.pop(connection.node_id, None) self.preferred.pop(connection.endpoint, None) self.connections.pop(connection.endpoint, None) self.outdated.pop(connection.endpoint, None) @@ -238,9 +274,11 @@ def execute_discovery(self) -> bool: self._cache.make_outdated(cached_endpoint) local_dc = resolve_details.self_location + priorities = resolve_details.bridge_priorities(True) if self._driver_config.prefer_primary_pile else None + self._cache.set_bridge_priorities(priorities) # Detect local DC using TCP latency if enabled and preferred is meaningful - if self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: + if priorities is None and self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: # Use only endpoints that match the SSL requirements for detection ssl_filtered_endpoints = [ endpoint @@ -291,7 +329,10 @@ def execute_discovery(self) -> bool: endpoint, endpoint_options, ) in resolved_endpoint.endpoints_with_options(): - if self._cache.size >= self._max_size or self._cache.already_exists(endpoint): + if ( + self._cache.already_exists(endpoint) + or not self._cache.connection_slot(endpoint, self._max_size)[0] + ): continue ready_connection = connection_impl.Connection.ready_factory( @@ -300,7 +341,14 @@ def execute_discovery(self) -> bool: self._ready_timeout, endpoint_options=endpoint_options, ) - self._cache.add(ready_connection, preferred) + if ready_connection is not None: + allowed, replacement = self._cache.connection_slot(endpoint, self._max_size) + if allowed: + if replacement is not None: + self._cache.make_outdated(replacement) + self._cache.add(ready_connection, preferred) + else: + ready_connection.close() self._cache.cleanup_outdated() diff --git a/ydb/resolver.py b/ydb/resolver.py index 40a049807..fd66108db 100644 --- a/ydb/resolver.py +++ b/ydb/resolver.py @@ -7,14 +7,16 @@ import random import itertools import typing -from typing import Any, ContextManager, List, Optional, Iterator +from typing import Any, ContextManager, Dict, List, Optional, Iterator from . import connection as conn_impl, driver, issues, settings as settings_impl, _apis # Workaround for good IDE and universal for runtime if typing.TYPE_CHECKING: from ._grpc.v4.protos import ydb_discovery_pb2 + from ._grpc.v4.protos import ydb_bridge_common_pb2 else: from ._grpc.common.protos import ydb_discovery_pb2 + from ._grpc.common.protos import ydb_bridge_common_pb2 logger = logging.getLogger(__name__) @@ -31,6 +33,7 @@ class EndpointInfo(object): "ipv6_addrs", "ssl_target_name_override", "node_id", + "bridge_pile_name", ) def __init__(self, endpoint_info: ydb_discovery_pb2.EndpointInfo): @@ -43,6 +46,7 @@ def __init__(self, endpoint_info: ydb_discovery_pb2.EndpointInfo): self.ipv6_addrs = tuple(endpoint_info.ip_v6) self.ssl_target_name_override = endpoint_info.ssl_target_name_override self.node_id = endpoint_info.node_id + self.bridge_pile_name = endpoint_info.bridge_pile_name def endpoints_with_options(self) -> typing.Generator[typing.Tuple[str, conn_impl.EndpointOptions], None, None]: ssl_target_name_override = None @@ -95,9 +99,27 @@ def _list_endpoints_request_factory(connection_params: driver.DriverConfig) -> _ class DiscoveryResult(object): - def __init__(self, self_location: str, endpoints: "list[EndpointInfo]"): + def __init__( + self, self_location: str, endpoints: "list[EndpointInfo]", pile_states: Optional[Dict[str, int]] = None + ): self.self_location = self_location self.endpoints = endpoints + self.pile_states = pile_states or {} + + def bridge_priorities(self, enabled: bool) -> Optional[Dict[str, int]]: + if not enabled or not self.pile_states: + return None + priorities = {} + for info in self.endpoints: + state = self.pile_states.get(info.bridge_pile_name, ydb_bridge_common_pb2.PileState.UNSPECIFIED) + priority = ( + 0 + if state in (ydb_bridge_common_pb2.PileState.PRIMARY, ydb_bridge_common_pb2.PileState.PROMOTED) + else (1 if state == ydb_bridge_common_pb2.PileState.SYNCHRONIZED else 2) + ) + for endpoint, _ in info.endpoints_with_options(): + priorities[endpoint] = priority + return priorities def __str__(self): return "DiscoveryResult " % ( @@ -139,7 +161,7 @@ def from_response( result.extend(local_endpoints) result.extend(different_endpoints) - return cls(message.self_location, result) + return cls(message.self_location, result, {pile.pile_name: pile.state for pile in message.pile_states}) class DiscoveryEndpointsResolver(object): From 7c9a56564072e549cb6f5be84d1d873fef32bd40 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Wed, 11 Nov 2026 19:21:08 +0300 Subject: [PATCH 2/6] Match asyncio timeout errors in endpoint tests --- tests/aio/test_discovery_detect_local_dc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/aio/test_discovery_detect_local_dc.py b/tests/aio/test_discovery_detect_local_dc.py index a090c6d39..ca89c1a28 100644 --- a/tests/aio/test_discovery_detect_local_dc.py +++ b/tests/aio/test_discovery_detect_local_dc.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import asyncio import pytest from unittest.mock import MagicMock, patch, AsyncMock from ydb import driver, connection @@ -157,7 +158,7 @@ def __init__(self, endpoint, driver_config, endpoint_options=None): async def connection_ready(self, ready_timeout): if self.endpoint in self.failed: - raise TimeoutError() + raise asyncio.TimeoutError() def add_cleanup_callback(self, callback): self.callbacks.append(callback) @@ -311,8 +312,6 @@ async def test_primary_driver_keyword_and_config_override(monkeypatch): @pytest.mark.asyncio async def test_primary_discovery_cancellation_closes_candidate(monkeypatch): - import asyncio - discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) monkeypatch.setattr(PrimaryConnection, "connection_ready", AsyncMock(side_effect=asyncio.CancelledError)) with pytest.raises(asyncio.CancelledError): From add6db9a153f63381c22c165b5f73b1e1f446bcf Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Thu, 12 Nov 2026 11:40:24 +0300 Subject: [PATCH 3/6] Qualify primary pile routing on a live hosted Bridge cluster --- .github/workflows/slo-2dc.yml | 118 +++++++++++++ tests/slo/README.md | 30 ++++ tests/slo/bridge/.gitignore | 1 + tests/slo/bridge/checks.py | 138 ++++++++++++++++ tests/slo/bridge/prepare.py | 85 ++++++++++ tests/slo/bridge/switchover.sh | 58 +++++++ tests/slo/bridge/verify.py | 184 +++++++++++++++++++++ tests/slo/requirements.txt | 2 +- tests/slo/src/bridge_observer.py | 61 +++++++ tests/slo/src/options.py | 1 + tests/slo/src/primary_probe.py | 209 ++++++++++++++++++++++++ tests/slo/src/root_runner.py | 2 + tests/test_discovery_detect_local_dc.py | 114 +++++++++++++ 13 files changed, 1002 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/slo-2dc.yml create mode 100644 tests/slo/bridge/.gitignore create mode 100644 tests/slo/bridge/checks.py create mode 100644 tests/slo/bridge/prepare.py create mode 100644 tests/slo/bridge/switchover.sh create mode 100644 tests/slo/bridge/verify.py create mode 100644 tests/slo/src/bridge_observer.py create mode 100644 tests/slo/src/primary_probe.py diff --git a/.github/workflows/slo-2dc.yml b/.github/workflows/slo-2dc.yml new file mode 100644 index 000000000..f0e50aef7 --- /dev/null +++ b/.github/workflows/slo-2dc.yml @@ -0,0 +1,118 @@ +name: Hosted 2DC SLO + +on: + pull_request: + paths: + - .github/workflows/slo-2dc.yml + - tests/slo/bridge/** + - tests/slo/** + - ydb/driver.py + - ydb/resolver.py + - ydb/pool.py + - ydb/aio/driver.py + - ydb/aio/pool.py + workflow_dispatch: + inputs: + server_image: + description: YDB server image to qualify (resolved to a digest) + default: cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb:trunk + required: true + +permissions: + contents: read + +jobs: + bridge: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + concurrency: + group: hosted-2dc-${{ github.ref }} + cancel-in-progress: true + env: + WORKLOAD_MEMORY_LIMIT: 1G + WORKLOAD_CPU_LIMIT: '1.00' + WORKLOAD_CURRENT_IMAGE: py-slo-bridge:current + REQUESTED_IMAGE: ${{ inputs.server_image || 'cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb@sha256:bce9caa6a1a6aac9b513510b46f61c4d4b58c1f88d937bcb54c20dc1bda6c8d4' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: ydb-platform/ydb-slo-action + ref: 54c3c03de1ce8c6a2d8b0828b328bd5db82ef2bb + path: .slo-action + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install the Compose version required by the action + run: | + set -euo pipefail + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL --retry 3 \ + https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \ + -o "$HOME/.docker/cli-plugins/docker-compose" + chmod +x "$HOME/.docker/cli-plugins/docker-compose" + docker compose version + docker compose up --help | grep -F -- --quiet-build + - name: Record runner and resolve server image + run: | + set -euo pipefail + mkdir -p bridge-artifacts + free -h + df -h + docker version + docker compose version + docker pull --platform linux/amd64 "$REQUESTED_IMAGE" + docker image inspect "$REQUESTED_IMAGE" > bridge-artifacts/server-image.json + SERVER_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$REQUESTED_IMAGE") + printf '%s\n' "$SERVER_IMAGE" > bridge-artifacts/server-image.txt + python tests/slo/bridge/prepare.py .slo-action "$SERVER_IMAGE" + - name: Build current Python SDK workload + run: docker build -f tests/slo/Dockerfile -t py-slo-bridge:current . + - name: Record node events and stop early on a fatal probe or chaos error + run: | + docker events --filter type=container --filter label=ydb.node.type=database \ + --filter event=stop --filter event=start --format '{{json .}}' \ + > bridge-artifacts/docker-events.jsonl 2>&1 & + ( + while sleep 5; do + native=$(docker inspect --format '{{.State.Status}}' ydb-workload-current 2>/dev/null || true) + [ "$native" != exited ] || exit 0 + for container in ydb-bridge-probe ydb-chaos-monkey; do + state=$(docker inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$container" 2>/dev/null || true) + case "$state" in + exited:0|'') ;; + exited:*) docker kill ydb-workload-current >/dev/null 2>&1 || true; exit 0 ;; + esac + done + done + ) > bridge-artifacts/watchdog.log 2>&1 & + - name: Run all five Bridge scenarios + id: slo + uses: ./.slo-action/init + timeout-minutes: 50 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workload_name: sync-query + workload_duration: '1500' + workload_current_ref: current + workload_current_image: py-slo-bridge:current + workload_current_command: >- + --read-rps 50 --write-rps 10 --read-threads 2 --write-threads 2 + --read-timeout 30000 --write-timeout 30000 + --prefer-primary-pile + bridge_mode: true + disable_compose_profiles: extra-nodes + fail_on_workload_error: true + - name: Verify scenario coverage, SDK results and data + if: always() + timeout-minutes: 3 + env: + ACTION_OUTCOME: ${{ steps.slo.outcome }} + run: python tests/slo/bridge/verify.py + - name: Upload qualification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: python-hosted-2dc + path: bridge-artifacts/ + retention-days: 14 diff --git a/tests/slo/README.md b/tests/slo/README.md index 88d46e724..816992948 100644 --- a/tests/slo/README.md +++ b/tests/slo/README.md @@ -1,5 +1,35 @@ # SLO workload +## GitHub-hosted PRIMARY pile qualification + +`.github/workflows/slo-2dc.yml` runs the pinned SLO Action v2 on `ubuntu-24.04`, +using two storage and two database nodes in a real Bridge cluster. It is a ready-made +Bridge topology with `erasure=none` and SectorMap disks, not the config-v1 migration procedure. +The image is pinned by digest and its OCI metadata is retained with the results. + +The 25-minute native Query workload enables `--prefer-primary-pile`. A separate probe +keeps one driver per sync/async Table/Query mode, creates fresh sessions, writes and reads +unique transactional rows, and runs an additional disabled-option control driver. +An independent raw discovery RPC maps server-reported session node IDs to pile roles; +the SDK's cache or priority implementation is not used as the oracle. + +The first scenario stops only PRIMARY compute for 45 seconds to exercise synchronized +fallback. It then holds A, B and A as PRIMARY for 160 seconds each, without node restarts +between these observation windows. Default discovery intervals are retained. In each +window's last 30 seconds every enabled mode must create at least three sessions on PRIMARY +and confirm at least three writes; the disabled control must use both piles. + +All five upstream switchover/takedown/failover scenarios run. Every enabled mode must +resume data operations after every recovery and continue near the end. A separate CLI +query audits every acknowledged row. Transient operation errors are recorded and allowed; +wrong-pile routing after stabilization, stalled modes, lost/corrupt data, OOM, process +restarts or incomplete chaos fail the job. The oracle, workload, node events and audit +are retained in the `python-hosted-2dc` artifact for 14 days. + +The runner adapter bounds resources and cold-start readiness time, restricts bootstrap +DNS aliases to the two active compute nodes, and extends +the first scenario's observation windows; the action lifecycle remains at the pinned revision. + SLO is the type of test where app based on ydb-sdk is tested against falling YDB cluster nodes, tablets, network (that is possible situations for distributed DBs with hundreds of nodes) diff --git a/tests/slo/bridge/.gitignore b/tests/slo/bridge/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/tests/slo/bridge/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tests/slo/bridge/checks.py b/tests/slo/bridge/checks.py new file mode 100644 index 000000000..a84f4707a --- /dev/null +++ b/tests/slo/bridge/checks.py @@ -0,0 +1,138 @@ +"""Acceptance checks for independently observed live Bridge routing and data.""" + +from datetime import datetime +import re + +MODES = ("sync-table", "sync-query", "async-table", "async-query") + + +def stamp(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + + +def check_routing(log, events, stopped_at): + errors, windows, beginnings = [], [], {} + pattern = r"\[([^\]]+)\] PRIMARY_WINDOW_(BEGIN|END) (\w+) (\S+)" + for when, kind, name, pile in re.findall(pattern, log): + if kind == "BEGIN": + beginnings[name] = (stamp(when), pile) + elif name in beginnings: + start, initial_pile = beginnings.pop(name) + windows.append({"name": name, "start": start, "end": stamp(when), "pile": pile}) + if pile != initial_pile or stamp(when) - start < 150: + errors.append(f"Invalid PRIMARY observation window: {name}") + if [w["name"] for w in windows] != ["initial", "switched", "restored"] or beginnings: + errors.append("Missing the three controlled PRIMARY observation windows") + elif not (windows[0]["pile"] == windows[2]["pile"] != windows[1]["pile"]): + errors.append("PRIMARY did not switch to the other pile and back") + details = {"primary_windows": [], "recovery_windows": [], "fallback": {}} + for window in windows: + end, primary = window["end"], window["pile"] + late = [row for row in events if end - 30 <= row.get("ts", 0) <= end] + routes = [ + row + for row in late + if row.get("event") == "session_created" + and row.get("stable") + and row.get("both_piles_visible") + and row.get("primary") == primary + ] + observation = {**window, "modes": {}} + for mode in MODES: + samples = [row for row in routes if row.get("mode") == mode] + writes = sum(row.get("event") == "operation" and row.get("ok") and row.get("mode") == mode for row in late) + wrong = [row for row in samples if row.get("pile") != primary] + observation["modes"][mode] = {"sessions": len(samples), "wrong_pile": len(wrong), "writes": writes} + if len(samples) < 3 or writes < 3 or wrong: + errors.append(f"PRIMARY routing or data progress failed for {mode} in {window['name']}") + control = {row.get("pile") for row in routes if row.get("mode") == "control-table"} + observation["control_piles"] = sorted(pile for pile in control if pile) + if control != {"pile-1", "pile-2"}: + errors.append(f"Disabled-option control did not use both piles in {window['name']}") + details["primary_windows"].append(observation) + fallback = re.findall(r"\[([^\]]+)\] PRIMARY_FALLBACK_(BEGIN|END) (\S+)", log) + if len(fallback) != 2 or [item[1] for item in fallback] != ["BEGIN", "END"] or fallback[0][2] != fallback[1][2]: + errors.append("Missing controlled primary-compute outage") + else: + start, end = stamp(fallback[0][0]), stamp(fallback[1][0]) + for mode in MODES: + count = sum( + row.get("event") == "session_created" + and row.get("mode") == mode + and start + 2 <= row.get("ts", 0) <= end + and row.get("stable") + and row.get("fallback") + and row.get("primary") == fallback[0][2] + for row in events + ) + details["fallback"][mode] = count + if not count: + errors.append(f"No synchronized fallback observed for {mode} while PRIMARY compute was stopped") + recoveries = re.findall(r"\[([^\]]+)\] Recovered: ([^\n]+)", log) + if len(recoveries) != 5: + errors.append(f"Expected five completed chaos recoveries, found {len(recoveries)}") + for when, name in recoveries: + start = stamp(when) + counts = { + mode: sum( + row.get("event") == "operation" + and row.get("ok") + and row.get("mode") == mode + and start <= row.get("ts", 0) <= start + 90 + for row in events + ) + for mode in MODES + } + details["recovery_windows"].append({"name": name, "writes": counts}) + if any(count < 3 for count in counts.values()): + errors.append(f"Some probe modes did not recover after {name}") + details["final_writes"] = { + mode: sum( + row.get("event") == "operation" + and row.get("ok") + and row.get("mode") == mode + and stopped_at - 60 <= row.get("ts", 0) <= stopped_at + for row in events + ) + for mode in MODES + } + if any(count < 3 for count in details["final_writes"].values()): + errors.append("Some probe modes stopped making progress at the end of the run") + return details, errors + + +def audit_rows(probe, rows): + stored = {(row["mode"], int(row["seq"])): row["payload"] for row in rows} + return [ + {"mode": mode, "seq": seq} + for mode, result in probe["modes"].items() + for seq in result["confirmed"] + if stored.get((mode, seq)) != f"{probe['table']}:{mode}:{seq}" + ] + + +def check_node_events(log, events): + windows = re.findall(r"\[([^\]]+)\] PRIMARY_WINDOW_(BEGIN|END) (\w+) (\S+)", log) + if len(windows) != 6: + return ["Missing node-event observation boundaries"] + first, last = stamp(windows[0][0]), stamp(windows[-1][0]) + errors = [] + for event in events: + when = event["timeNano"] / 1e9 if event.get("timeNano") else event["time"] + if first <= when <= last: + errors.append("A database process changed during the role-only observation windows") + break + fallback = re.findall(r"\[([^\]]+)\] PRIMARY_FALLBACK_(BEGIN|END) (\S+)", log) + if len(fallback) != 2: + return errors + ["Missing controlled outage node boundaries"] + start, end, pile = stamp(fallback[0][0]), stamp(fallback[1][0]), fallback[0][2] + stops = [ + event + for event in events + if (event.get("Action") or event.get("status")) == "stop" + and event.get("Actor", {}).get("Attributes", {}).get("ydb.node.location.pile") == pile + and start <= (event["timeNano"] / 1e9 if event.get("timeNano") else event["time"]) <= end + ] + if not stops: + errors.append("No Docker evidence that PRIMARY compute actually stopped") + return errors diff --git a/tests/slo/bridge/prepare.py b/tests/slo/bridge/prepare.py new file mode 100644 index 000000000..92e01bd56 --- /dev/null +++ b/tests/slo/bridge/prepare.py @@ -0,0 +1,85 @@ +"""Configure a pinned Action checkout for a bounded GitHub-hosted run.""" + +import argparse +import copy +import json +import subprocess +import shutil +from pathlib import Path + + +def prepare(action, image): + deploy = action.resolve() / "deploy" + compose = deploy / "compose.bridge.yml" + rendered = subprocess.check_output( + [ + "docker", + "compose", + "-f", + str(compose), + "config", + "--no-interpolate", + "--format", + "json", + ], + text=True, + ) + config = json.loads(rendered) + for service in config["services"].values(): + if "build" in service and service["build"]["context"].endswith("/ydb"): + service["build"].setdefault("args", {})["YDB_IMAGE"] = image + service["mem_limit"] = "2g" + for name in ("workload-current", "workload-baseline"): + config["services"][name]["deploy"]["resources"]["reservations"]["memory"] = "256m" + service = config["services"][name] + hosts = service.get("extra_hosts", []) + if isinstance(hosts, dict): + hosts = [ + f"{host}:{address}" + for host, addresses in hosts.items() + for address in (addresses if isinstance(addresses, list) else [addresses]) + ] + service["extra_hosts"] = [host for host in hosts if not host.startswith(("ydb:", "ydb="))] + [ + "ydb:172.28.0.11", + "ydb:172.28.0.12", + ] + for filename in ("config.yaml", "config.bridge.yaml"): + path = deploy / "ydb/rootfs/opt/ydb.tech/ydbd/cfg" / filename + text = path.read_text() + if text.count("cpu_count: 4") != 1 or text.count("hard_limit_bytes: 17179869184") != 1: + raise RuntimeError(f"Unexpected resource configuration: {path}") + path.write_text( + text.replace("cpu_count: 4", "cpu_count: 1").replace( + "hard_limit_bytes: 17179869184", "hard_limit_bytes: 1610612736" + ) + ) + readiness = deploy / "ydb/rootfs/opt/ydb.tech/scripts/ydbd/check-readiness.sh" + text = readiness.read_text() + if text.count("local max_attempts=30") != 2: + raise RuntimeError("Unexpected readiness implementation") + readiness.write_text(text.replace("local max_attempts=30", "local max_attempts=150", 1)) + scenario = deploy / "chaos/rootfs/opt/ydb.tech/chaos/bridge-scenarios/01-switchover.sh" + shutil.copyfile(Path(__file__).with_name("switchover.sh"), scenario) + scenario.chmod(0o755) + probe = copy.deepcopy(config["services"]["workload-current"]) + probe.update( + container_name="ydb-bridge-probe", entrypoint=["python", "/src/tests/slo/src/primary_probe.py"], command=[] + ) + probe.pop("deploy", None) + probe["mem_limit"] = "768m" + probe["restart"] = "no" + environment = probe.get("environment", {}) + if isinstance(environment, list): + environment = {item.partition("=")[0]: item.partition("=")[2] if "=" in item else None for item in environment} + probe["environment"] = {**environment, "ORACLE_ENDPOINTS": "ydb-storage-1:2136,ydb-storage-2:2136"} + config["services"]["bridge-probe"] = probe + compose.write_text(json.dumps(config, indent=2) + "\n") + subprocess.run(["docker", "compose", "-f", str(compose), "config", "--quiet"], check=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("action", type=Path) + parser.add_argument("image") + args = parser.parse_args() + prepare(args.action, args.image) diff --git a/tests/slo/bridge/switchover.sh b/tests/slo/bridge/switchover.sh new file mode 100644 index 000000000..626dcd16b --- /dev/null +++ b/tests/slo/bridge/switchover.sh @@ -0,0 +1,58 @@ +#!/bin/sh +set -e +. /opt/ydb.tech/scripts/chaos/libchaos.sh + +recovery_timeout="${BRIDGE_RECOVERY_TIMEOUT:-180}" +original_primary=$(get_bridge_pile_by_state PRIMARY) +new_primary=$(get_bridge_pile_by_state SYNCHRONIZED) +compute=$(get_bridge_pile_nodes "$original_primary" database) +fault_active=false + +cleanup() { + set +e + docker start "$compute" >/dev/null + restore_bridge_primary "$original_primary" "$recovery_timeout" + if [ "$fault_active" = true ]; then + chaos_recover "bridge-switchover" "$new_primary" "Original PRIMARY restored" + fi +} + +[ -n "$original_primary" ] && [ -n "$new_primary" ] && [ -n "$compute" ] +BRIDGE_CLI_PILE="$original_primary" +install_bridge_cleanup_traps + +deadline=$(($(date +%s) + 120)) +until docker exec ydb-bridge-probe test -f /tmp/bridge-probe-ready; do + [ "$(date +%s)" -lt "$deadline" ] || exit 1 + sleep 2 +done + +log "PRIMARY_FALLBACK_BEGIN $original_primary" +docker stop -t 0 "$compute" +sleep 45 +log "PRIMARY_FALLBACK_END $original_primary" +docker start "$compute" >/dev/null +wait_container_healthy "$compute" "$recovery_timeout" + +log "PRIMARY_WINDOW_BEGIN initial $original_primary" +sleep 160 +log "PRIMARY_WINDOW_END initial $original_primary" + +chaos_inject "bridge-switchover" "$new_primary" +fault_active=true +bridge_cli switchover --new-primary "$new_primary" +wait_bridge_pile_state "$new_primary" PRIMARY "$recovery_timeout" +BRIDGE_CLI_PILE="$new_primary" +log "PRIMARY_WINDOW_BEGIN switched $new_primary" +sleep 160 +log "PRIMARY_WINDOW_END switched $new_primary" + +restore_bridge_primary "$original_primary" "$recovery_timeout" +BRIDGE_CLI_PILE="$original_primary" +wait_bridge_pile_state "$new_primary" SYNCHRONIZED "$recovery_timeout" +log "PRIMARY_WINDOW_BEGIN restored $original_primary" +sleep 160 +log "PRIMARY_WINDOW_END restored $original_primary" +chaos_recover "bridge-switchover" "$new_primary" "Original PRIMARY restored" +fault_active=false +clear_bridge_cleanup_traps diff --git a/tests/slo/bridge/verify.py b/tests/slo/bridge/verify.py new file mode 100644 index 000000000..02583adbd --- /dev/null +++ b/tests/slo/bridge/verify.py @@ -0,0 +1,184 @@ +"""Verify live role-aware routing, recovery and independently audited data.""" + +import json +import math +import os +from pathlib import Path +import re +import subprocess +import urllib.parse +import urllib.request + +from checks import MODES, audit_rows, check_node_events, check_routing + +SCENARIOS = ( + "01-switchover", + "02-takedown-synchronized", + "03-takedown-primary", + "04-failover-synchronized", + "05-failover-primary", +) + + +def command(*args, check=True, timeout=90): + return subprocess.run( + args, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=check, timeout=timeout + ).stdout + + +def json_lines(text): + rows = [] + for line in text.splitlines(): + try: + row = json.loads(line) + if isinstance(row, dict): + rows.append(row) + except json.JSONDecodeError: + continue + return rows + + +def main(): + artifacts = Path("bridge-artifacts") + artifacts.mkdir(exist_ok=True) + result = {"errors": [], "action_outcome": os.environ.get("ACTION_OUTCOME")} + for name in ("ydb-workload-current", "ydb-chaos-monkey"): + (artifacts / f"{name}.log").write_text(command("docker", "logs", name, check=False)) + try: + (artifacts / "healthcheck.json").write_text( + command( + "docker", + "exec", + "ydb-storage-1", + "/opt/ydb.tech/ydb/bin/ydb", + "-e", + "grpc://localhost:2136", + "monitoring", + "healthcheck", + "--format", + "json", + check=False, + timeout=30, + ) + ) + except Exception as error: + (artifacts / "healthcheck-error.txt").write_text(repr(error)) + try: + if result["action_outcome"] != "success": + result["errors"].append("Action did not complete successfully") + chaos_info = json.loads(command("docker", "inspect", "ydb-chaos-monkey"))[0] + chaos = chaos_info["State"] + log = (artifacts / "ydb-chaos-monkey.log").read_text() + if ( + chaos["Status"] != "exited" + or chaos["ExitCode"] != 0 + or chaos_info["RestartCount"] + or "Chaos Monkey finished" not in log + ): + result["errors"].append("Chaos sequence did not finish without a process restart") + for scenario in SCENARIOS: + if f"Running scenario: {scenario}" not in log: + result["errors"].append(f"Scenario not executed: {scenario}") + state = json.loads(command("docker", "inspect", "ydb-bridge-probe"))[0] + if state["State"]["Running"]: + command("docker", "exec", "ydb-bridge-probe", "touch", "/tmp/bridge-probe-stop") + command("docker", "wait", "ydb-bridge-probe", timeout=150) + probe_log = command("docker", "logs", "ydb-bridge-probe", check=False) + (artifacts / "ydb-bridge-probe.log").write_text(probe_log) + command("docker", "cp", "ydb-bridge-probe:/tmp/bridge-probe.json", str(artifacts / "probe.json")) + probe = json.loads((artifacts / "probe.json").read_text()) + result["probe"] = probe + if not probe.get("passed") or state["RestartCount"]: + result["errors"].append("Probe failed or its process was restarted") + events = json_lines(probe_log) + for mode in (*MODES, "control-table"): + if sum(row.get("event") == "driver_started" and row.get("mode") == mode for row in events) != 1: + result["errors"].append(f"Expected one long-lived driver for {mode}") + routing, errors = check_routing(log, events, probe["stopped_at"]) + result["routing"] = routing + result["errors"].extend(errors) + result["errors"].extend(check_node_events(log, json_lines((artifacts / "docker-events.jsonl").read_text()))) + if not re.fullmatch(r"primary_probe_[a-f0-9]+", probe["table"]): + raise ValueError("Unexpected audit table name") + output = command( + "docker", + "exec", + "ydb-database-1", + "/opt/ydb.tech/ydb/bin/ydb", + "-e", + "grpc://localhost:2136", + "-d", + "/Root/testdb", + "--no-discovery", + "sql", + "-s", + f"SELECT `mode`, seq, payload FROM `{probe['table']}`;", + "--format", + "json-unicode", + ) + (artifacts / "audit-rows.jsonl").write_text(output) + missing = audit_rows(probe, json_lines(output)) + (artifacts / "missing.json").write_text(json.dumps(missing, indent=2)) + result["audit"] = { + "confirmed": sum(len(item["confirmed"]) for item in probe["modes"].values()), + "missing": len(missing), + } + if missing: + result["errors"].append("Acknowledged probe rows were lost or corrupted") + ip = command( + "docker", + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + "ydb-prometheus", + ).strip() + + def metric(expression): + url = f"http://{ip}:9090/api/v1/query?" + urllib.parse.urlencode({"query": expression}) + with urllib.request.urlopen(url, timeout=15) as response: + data = json.load(response) + if data["status"] != "success": + raise RuntimeError(data) + return sum(float(item["value"][1]) for item in data["data"]["result"]) + + observations = {} + for operation in ("read", "write"): + selector = f'sdk_operations_success_total{{ref="current",operation_type="{operation}"}}' + count, recent = metric(f"sum({selector})"), metric(f"sum(increase({selector}[120s]))") + observations[operation], observations[f"recent_{operation}"] = count, recent + if not math.isfinite(count) or not math.isfinite(recent) or count <= 0 or recent <= 0: + result["errors"].append(f"No recent successful native {operation} observations") + observations["errors"] = metric('sum(sdk_operations_failure_total{ref="current"})') + result["native_observations"] = observations + containers = json.loads( + command( + "docker", + "inspect", + "ydb-storage-1", + "ydb-storage-2", + "ydb-database-1", + "ydb-database-2", + "ydb-workload-current", + "ydb-bridge-probe", + ) + ) + result["container_states"] = { + node["Name"]: {"state": node["State"], "restarts": node["RestartCount"]} for node in containers + } + if any(node["State"]["OOMKilled"] or node["RestartCount"] for node in containers): + result["errors"].append("A container was OOM killed or unexpectedly restarted") + for node in containers: + if node["Name"] in ("/ydb-workload-current", "/ydb-bridge-probe") and ( + node["State"]["Status"] != "exited" or node["State"]["ExitCode"] != 0 + ): + result["errors"].append(f"Workload process did not exit successfully: {node['Name']}") + except Exception as error: + result["errors"].append(repr(error)) + result["passed"] = not result["errors"] + (artifacts / "result.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/slo/requirements.txt b/tests/slo/requirements.txt index 68f516f13..d9021cc5c 100644 --- a/tests/slo/requirements.txt +++ b/tests/slo/requirements.txt @@ -4,6 +4,6 @@ hdrhistogram==0.10.3 # OpenTelemetry (OTLP/HTTP exporter) # NOTE: OpenTelemetry 1.39.1 requires Python >= 3.9. -opentelemetry-api==1.45.1 +opentelemetry-api==1.39.1 opentelemetry-sdk==1.39.1 opentelemetry-exporter-otlp-proto-http==1.39.1 diff --git a/tests/slo/src/bridge_observer.py b/tests/slo/src/bridge_observer.py new file mode 100644 index 000000000..1a52dc8e1 --- /dev/null +++ b/tests/slo/src/bridge_observer.py @@ -0,0 +1,61 @@ +"""Read the server's discovery RPC directly, independently of driver balancing.""" + +import grpc +import ydb +from ydb import _apis + + +class BridgeObserver: + def __init__(self, endpoints, database): + self.database = database + self.channels = [grpc.insecure_channel(endpoint) for endpoint in endpoints] + self.stubs = [_apis.DiscoveryService.Stub(channel) for channel in self.channels] + + def snapshot(self): + for stub in self.stubs: + try: + response = stub.ListEndpoints( + _apis.ydb_discovery.ListEndpointsRequest(database=self.database), timeout=2 + ) + if response.operation.status != ydb.StatusCode.SUCCESS: + continue + result = _apis.ydb_discovery.ListEndpointsResult() + if not response.operation.result.Unpack(result): + continue + states = {pile.pile_name: pile.state for pile in result.pile_states} + primary = next((name for name, state in states.items() if state == 1), None) + return { + "states": states, + "primary": primary, + "node_piles": {str(ep.node_id): ep.bridge_pile_name for ep in result.endpoints if not ep.ssl}, + } + except grpc.RpcError: + continue + return None + + def close(self): + for channel in self.channels: + channel.close() + + +def route_observation(node_id, before, after): + before, after = before or {}, after or {} + key = str(node_id) + pile = after.get("node_piles", {}).get(key) + stable = bool( + before.get("primary") + and before.get("primary") == after.get("primary") + and before.get("states") == after.get("states") + and pile + and before.get("node_piles", {}).get(key) == pile + ) + return { + "node_id": node_id, + "pile": pile, + "primary": after.get("primary"), + "stable": stable, + "both_piles_visible": {"pile-1", "pile-2"} <= set(after.get("node_piles", {}).values()), + "fallback": stable and after["states"].get(pile) == 3 and pile != after["primary"], + "before": before, + "after": after, + } diff --git a/tests/slo/src/options.py b/tests/slo/src/options.py index 200c3c556..08e33d3db 100644 --- a/tests/slo/src/options.py +++ b/tests/slo/src/options.py @@ -7,6 +7,7 @@ def add_common_options(parser): parser.add_argument("-t", "--table-name", default="key_value", help="Table name") parser.add_argument("--debug", action="store_true", help="Enable debug logging") parser.add_argument("--async", action="store_true", help="Use async mode for operations") + parser.add_argument("--prefer-primary-pile", action="store_true", help="Prefer PRIMARY/PROMOTED Bridge endpoints") def make_table_create_parser(subparsers): diff --git a/tests/slo/src/primary_probe.py b/tests/slo/src/primary_probe.py new file mode 100644 index 000000000..1efeffbee --- /dev/null +++ b/tests/slo/src/primary_probe.py @@ -0,0 +1,209 @@ +"""Exercise fresh-session routing and acknowledged data on long-lived drivers.""" + +import asyncio +import json +import os +from pathlib import Path +import threading +import time +import traceback +from urllib.parse import parse_qs, urlparse +import uuid + +import ydb +import ydb.aio + +from bridge_observer import BridgeObserver, route_observation + +MODES = ("sync-table", "sync-query", "async-table", "async-query", "control-table") + + +class Probe: + def __init__(self): + self.table = "primary_probe_" + uuid.uuid4().hex + self.stop = threading.Event() + self.lock = threading.Lock() + self.stats = {mode: {"successes": 0, "errors": [], "confirmed": [], "driver_starts": 0} for mode in MODES} + self.fatal = [] + self.observer = BridgeObserver(os.environ["ORACLE_ENDPOINTS"].split(","), os.environ["YDB_DATABASE"]) + + def emit(self, event, mode, **values): + with self.lock: + print(json.dumps({"event": event, "mode": mode, "ts": time.time(), **values}), flush=True) + + def error(self, mode, error, stage): + with self.lock: + self.stats[mode]["errors"].append({"stage": stage, "error": repr(error)}) + self.emit("error", mode, error=repr(error), stage=stage) + + def fail(self, mode, error): + with self.lock: + self.fatal.append({"mode": mode, "error": repr(error)}) + traceback.print_exc() + self.stop.set() + + def config(self, mode): + return ydb.DriverConfig( + os.environ["YDB_ENDPOINT"], + database=os.environ["YDB_DATABASE"], + credentials=ydb.AnonymousCredentials(), + prefer_primary_pile=mode != "control-table", + ) + + def sql(self, mode, seq): + payload = f"{self.table}:{mode}:{seq}" + return ( + f"UPSERT INTO `{self.table}` (`mode`, seq, payload) VALUES " + f'(CAST("{mode}" AS Utf8), CAST({seq} AS Uint64), CAST("{payload}" AS Utf8)); ' + f'SELECT payload FROM `{self.table}` WHERE `mode` = CAST("{mode}" AS Utf8) ' + f"AND seq = CAST({seq} AS Uint64);" + ) + + def validate(self, mode, seq, rows): + if len(rows) != 1 or len(rows[0].rows) != 1 or rows[0].rows[0].payload != f"{self.table}:{mode}:{seq}": + raise AssertionError(f"Incorrect read-after-write for {mode}/{seq}") + + def confirm(self, mode, seq): + with self.lock: + self.stats[mode]["successes"] += 1 + self.stats[mode]["confirmed"].append(seq) + if all(self.stats[name]["successes"] for name in MODES): + Path("/tmp/bridge-probe-ready").touch() + self.emit("operation", mode, seq=seq, ok=True) + + def started(self, mode): + with self.lock: + self.stats[mode]["driver_starts"] += 1 + self.emit("driver_started", mode, prefer_primary_pile=mode != "control-table", sdk=ydb.__version__) + + def session_created(self, mode, session, before, after): + if mode.endswith("query"): + node_id = session.node_id + else: + node_id = int(parse_qs(urlparse(session.session_id).query)["node_id"][0]) + self.emit("session_created", mode, **route_observation(node_id, before, after)) + + def sync_worker(self, mode): + try: + with ydb.Driver(self.config(mode)) as driver: + driver.wait(timeout=60) + self.started(mode) + seq = 0 + while not self.stop.is_set(): + seq += 1 + session = None + try: + before = self.observer.snapshot() + session = ydb.QuerySession(driver) if mode.endswith("query") else driver.table_client.session() + session.create(settings=ydb.BaseRequestSettings().with_timeout(5)) + self.session_created(mode, session, before, self.observer.snapshot()) + if mode.endswith("query"): + with session.transaction(ydb.QuerySerializableReadWrite()) as tx: + with tx.execute( + self.sql(mode, seq), settings=ydb.BaseRequestSettings().with_timeout(5) + ) as stream: + self.validate(mode, seq, list(stream)) + tx.commit(settings=ydb.BaseRequestSettings().with_timeout(5)) + else: + rows = session.transaction(ydb.SerializableReadWrite()).execute( + self.sql(mode, seq), commit_tx=True, settings=ydb.BaseRequestSettings().with_timeout(5) + ) + self.validate(mode, seq, rows) + self.confirm(mode, seq) + except ydb.Error as error: + self.error(mode, error, "operation") + finally: + if session is not None and session.session_id: + try: + session.delete(settings=ydb.BaseRequestSettings().with_timeout(5)) + except ydb.Error as error: + self.error(mode, error, "cleanup") + self.stop.wait(1) + except BaseException as error: + self.fail(mode, error) + + async def async_worker(self, mode): + try: + async with ydb.aio.Driver(self.config(mode)) as driver: + await driver.wait(timeout=60) + self.started(mode) + seq = 0 + while not self.stop.is_set(): + seq += 1 + session = None + try: + before = await asyncio.to_thread(self.observer.snapshot) + session = ( + ydb.aio.QuerySession(driver) if mode.endswith("query") else driver.table_client.session() + ) + await session.create(settings=ydb.BaseRequestSettings().with_timeout(5)) + after = await asyncio.to_thread(self.observer.snapshot) + self.session_created(mode, session, before, after) + if mode.endswith("query"): + async with session.transaction(ydb.QuerySerializableReadWrite()) as tx: + async with await tx.execute( + self.sql(mode, seq), settings=ydb.BaseRequestSettings().with_timeout(5) + ) as stream: + self.validate(mode, seq, [part async for part in stream]) + await tx.commit(settings=ydb.BaseRequestSettings().with_timeout(5)) + else: + rows = await session.transaction(ydb.SerializableReadWrite()).execute( + self.sql(mode, seq), commit_tx=True, settings=ydb.BaseRequestSettings().with_timeout(5) + ) + self.validate(mode, seq, rows) + self.confirm(mode, seq) + except ydb.Error as error: + self.error(mode, error, "operation") + finally: + if session is not None and session.session_id: + try: + await session.delete(settings=ydb.BaseRequestSettings().with_timeout(5)) + except ydb.Error as error: + self.error(mode, error, "cleanup") + await asyncio.sleep(1) + except BaseException as error: + self.fail(mode, error) + + async def async_workers(self): + await asyncio.gather(self.async_worker("async-table"), self.async_worker("async-query")) + + def run(self): + with ydb.Driver(self.config("control-table")) as setup: + setup.wait(timeout=60) + with ydb.QuerySessionPool(setup) as pool: + version = pool.execute_with_retries("SELECT CAST(version() AS Utf8) AS version;")[0].rows[0].version + pool.execute_with_retries( + f"CREATE TABLE `{self.table}` (`mode` Utf8, seq Uint64, payload Utf8, PRIMARY KEY(`mode`, seq));" + ) + threads = [ + threading.Thread(target=self.sync_worker, args=(mode,), daemon=True) + for mode in ("sync-table", "sync-query", "control-table") + ] + threads.append(threading.Thread(target=lambda: asyncio.run(self.async_workers()), daemon=True)) + for thread in threads: + thread.start() + deadline = time.monotonic() + int(os.environ["WORKLOAD_DURATION"]) + 180 + while not self.stop.is_set() and time.monotonic() < deadline and not Path("/tmp/bridge-probe-stop").exists(): + time.sleep(0.5) + stopped_at = time.time() + self.stop.set() + for thread in threads: + thread.join(timeout=60) + if any(thread.is_alive() for thread in threads): + self.fatal.append({"error": "Probe worker did not finish"}) + self.observer.close() + result = { + "table": self.table, + "server_version": version, + "modes": self.stats, + "fatal": self.fatal, + "stopped_at": stopped_at, + "passed": not self.fatal and all(self.stats[mode]["successes"] for mode in MODES), + } + Path("/tmp/bridge-probe.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result), flush=True) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(Probe().run()) diff --git a/tests/slo/src/root_runner.py b/tests/slo/src/root_runner.py index 2c8c2e75a..5b7f5657e 100644 --- a/tests/slo/src/root_runner.py +++ b/tests/slo/src/root_runner.py @@ -49,6 +49,7 @@ def _run_sync_command(self, args, runner_instance, command): args.endpoint, database=args.db, grpc_keep_alive_timeout=5000, + prefer_primary_pile=getattr(args, "prefer_primary_pile", False), ) with ydb.Driver(driver_config) as driver: @@ -75,6 +76,7 @@ async def _run_async_command(self, args, runner_instance, command): args.endpoint, database=args.db, grpc_keep_alive_timeout=5000, + prefer_primary_pile=getattr(args, "prefer_primary_pile", False), ) async with ydb.aio.Driver(driver_config) as driver: diff --git a/tests/test_discovery_detect_local_dc.py b/tests/test_discovery_detect_local_dc.py index 6cea9eb2a..e36e301d1 100644 --- a/tests/test_discovery_detect_local_dc.py +++ b/tests/test_discovery_detect_local_dc.py @@ -280,3 +280,117 @@ def test_primary_driver_keyword_and_config_override(monkeypatch): assert seen[-1].prefer_primary_pile driver.Driver(config, prefer_primary_pile=False) assert not seen[-1].prefer_primary_pile + + +def slo_observations(): + from datetime import datetime, timezone + from tests.slo.bridge.checks import MODES + + base = datetime(2026, 1, 1, tzinfo=timezone.utc).timestamp() + log, events = [], [] + + def marker(second, text): + when = datetime.fromtimestamp(base + second, tz=timezone.utc).isoformat().replace("+00:00", "Z") + log.append(f"[{when}] {text}") + + marker(0, "PRIMARY_FALLBACK_BEGIN pile-1") + marker(20, "PRIMARY_FALLBACK_END pile-1") + for mode in MODES: + events.append( + { + "event": "session_created", + "mode": mode, + "ts": base + 10, + "stable": True, + "fallback": True, + "primary": "pile-1", + "pile": "pile-2", + } + ) + for name, start, pile in (("initial", 30, "pile-1"), ("switched", 200, "pile-2"), ("restored", 370, "pile-1")): + marker(start, f"PRIMARY_WINDOW_BEGIN {name} {pile}") + marker(start + 160, f"PRIMARY_WINDOW_END {name} {pile}") + for mode in MODES: + for offset in (140, 145, 150): + events.append( + { + "event": "session_created", + "mode": mode, + "ts": base + start + offset, + "stable": True, + "both_piles_visible": True, + "primary": pile, + "pile": pile, + } + ) + events.append({"event": "operation", "mode": mode, "ts": base + start + offset, "ok": True}) + for control_pile in ("pile-1", "pile-2"): + events.append( + { + "event": "session_created", + "mode": "control-table", + "ts": base + start + 145, + "stable": True, + "both_piles_visible": True, + "primary": pile, + "pile": control_pile, + } + ) + for second in (540, 650, 760, 870, 980): + marker(second, "Recovered: scenario (restored)") + for mode in MODES: + for offset in (1, 2, 3): + events.append({"event": "operation", "mode": mode, "ts": base + second + offset, "ok": True}) + for mode in MODES: + for second in (1090, 1091, 1092): + events.append({"event": "operation", "mode": mode, "ts": base + second, "ok": True}) + return "\n".join(log), events, base + 1100 + + +def test_live_primary_slo_acceptance_and_wrong_pile_rejection(): + from tests.slo.bridge.checks import check_routing + + log, events, stopped_at = slo_observations() + assert not check_routing(log, events, stopped_at)[1] + sample = next(row for row in events if row.get("both_piles_visible") and row["mode"] == "sync-table") + sample["pile"] = "pile-2" + assert any("sync-table" in error for error in check_routing(log, events, stopped_at)[1]) + + +def test_live_primary_slo_requires_control_and_all_modes(): + from tests.slo.bridge.checks import check_routing + + log, events, stopped_at = slo_observations() + events = [row for row in events if row["mode"] != "control-table"] + assert any("control" in error for error in check_routing(log, events, stopped_at)[1]) + events = [row for row in events if row["mode"] != "async-query"] + assert any( + "async-query" in error or "Some probe modes" in error for error in check_routing(log, events, stopped_at)[1] + ) + + +def test_live_primary_slo_independent_audit_and_docker_evidence(): + from tests.slo.bridge.checks import audit_rows, check_node_events + + probe = {"table": "primary_probe_abc", "modes": {"sync-table": {"confirmed": [1, 2]}}} + rows = [{"mode": "sync-table", "seq": seq, "payload": f"primary_probe_abc:sync-table:{seq}"} for seq in (1, 2)] + assert not audit_rows(probe, rows) + rows[1]["payload"] = "corrupted" + assert audit_rows(probe, rows) == [{"mode": "sync-table", "seq": 2}] + log, _, stopped_at = slo_observations() + base = stopped_at - 1100 + stop = {"time": base + 5, "Action": "stop", "Actor": {"Attributes": {"ydb.node.location.pile": "pile-1"}}} + assert not check_node_events(log, [stop]) + assert check_node_events(log, [stop, {"time": base + 250, "Action": "start"}]) + + +def test_live_primary_slo_oracle_and_cli_flag(): + from tests.slo.src.bridge_observer import route_observation + from tests.slo.src.options import get_root_parser + + before = {"states": {"pile-1": 1, "pile-2": 3}, "primary": "pile-1", "node_piles": {"1": "pile-1", "2": "pile-2"}} + assert route_observation(1, before, before)["stable"] + assert route_observation(2, before, before)["fallback"] + assert not route_observation(1, before, {**before, "primary": "pile-2"})["stable"] + args = get_root_parser().parse_args(["table-run", "grpc://host:2136", "/Root/testdb", "--prefer-primary-pile"]) + assert args.prefer_primary_pile From 7247c69b57c1adb251d1d6c19cd4ee1fbd522527 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Thu, 12 Nov 2026 12:12:07 +0300 Subject: [PATCH 4/6] Distinguish RPC cancellation from probe task cancellation --- tests/aio/test_discovery_detect_local_dc.py | 26 +++++++++++++ tests/slo/README.md | 5 +++ tests/slo/bridge/checks.py | 18 +++++++++ tests/slo/bridge/verify.py | 41 +++++++++++++-------- tests/slo/src/bridge_observer.py | 9 +++++ tests/slo/src/primary_probe.py | 11 +++++- tests/test_discovery_detect_local_dc.py | 11 ++++++ 7 files changed, 104 insertions(+), 17 deletions(-) diff --git a/tests/aio/test_discovery_detect_local_dc.py b/tests/aio/test_discovery_detect_local_dc.py index ca89c1a28..b71a587ca 100644 --- a/tests/aio/test_discovery_detect_local_dc.py +++ b/tests/aio/test_discovery_detect_local_dc.py @@ -318,3 +318,29 @@ async def test_primary_discovery_cancellation_closes_candidate(monkeypatch): await discovery.execute_discovery() assert len(PrimaryConnection.created) == 1 assert PrimaryConnection.created[0].closed + + +@pytest.mark.asyncio +async def test_live_slo_distinguishes_rpc_future_and_caller_task_cancellation(): + from tests.slo.src.bridge_observer import is_rpc_cancellation + + # Canceling an awaited future does not request cancellation of its caller task. + future = asyncio.get_running_loop().create_future() + future.cancel() + try: + await future + except asyncio.CancelledError: + assert is_rpc_cancellation() == hasattr(asyncio.current_task(), "cancelling") + + async def caller(): + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + assert not is_rpc_cancellation() + raise + + task = asyncio.create_task(caller()) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task diff --git a/tests/slo/README.md b/tests/slo/README.md index 816992948..a72eb137f 100644 --- a/tests/slo/README.md +++ b/tests/slo/README.md @@ -26,6 +26,11 @@ wrong-pile routing after stabilization, stalled modes, lost/corrupt data, OOM, p restarts or incomplete chaos fail the job. The oracle, workload, node events and audit are retained in the `python-hosted-2dc` artifact for 14 days. +Async gRPC may surface a canceled RPC future as `asyncio.CancelledError`. The probe +records it as an operation error only when no cancellation was requested for its +caller task. Caller-task cancellation is preserved. The workload image uses Python 3.11; +on runtimes without cancellation counters the probe conservatively propagates cancellation. + The runner adapter bounds resources and cold-start readiness time, restricts bootstrap DNS aliases to the two active compute nodes, and extends the first scenario's observation windows; the action lifecycle remains at the pinned revision. diff --git a/tests/slo/bridge/checks.py b/tests/slo/bridge/checks.py index a84f4707a..c38be7c85 100644 --- a/tests/slo/bridge/checks.py +++ b/tests/slo/bridge/checks.py @@ -98,6 +98,24 @@ def check_routing(log, events, stopped_at): } if any(count < 3 for count in details["final_writes"].values()): errors.append("Some probe modes stopped making progress at the end of the run") + details["final_routing"] = {} + if windows: + primary = windows[0]["pile"] + for mode in MODES: + samples = [ + row + for row in events + if row.get("event") == "session_created" + and row.get("mode") == mode + and stopped_at - 30 <= row.get("ts", 0) <= stopped_at + and row.get("stable") + and row.get("both_piles_visible") + and row.get("primary") == primary + ] + wrong = sum(row.get("pile") != primary for row in samples) + details["final_routing"][mode] = {"sessions": len(samples), "wrong_pile": wrong} + if len(samples) < 3 or wrong: + errors.append(f"PRIMARY routing did not recover at the end for {mode}") return details, errors diff --git a/tests/slo/bridge/verify.py b/tests/slo/bridge/verify.py index 02583adbd..972a5ac9e 100644 --- a/tests/slo/bridge/verify.py +++ b/tests/slo/bridge/verify.py @@ -100,22 +100,31 @@ def main(): result["errors"].extend(check_node_events(log, json_lines((artifacts / "docker-events.jsonl").read_text()))) if not re.fullmatch(r"primary_probe_[a-f0-9]+", probe["table"]): raise ValueError("Unexpected audit table name") - output = command( - "docker", - "exec", - "ydb-database-1", - "/opt/ydb.tech/ydb/bin/ydb", - "-e", - "grpc://localhost:2136", - "-d", - "/Root/testdb", - "--no-discovery", - "sql", - "-s", - f"SELECT `mode`, seq, payload FROM `{probe['table']}`;", - "--format", - "json-unicode", - ) + output = None + for node in ("ydb-database-1", "ydb-database-2"): + try: + output = command( + "docker", + "exec", + node, + "/opt/ydb.tech/ydb/bin/ydb", + "-e", + "grpc://localhost:2136", + "-d", + "/Root/testdb", + "--no-discovery", + "sql", + "-s", + f"SELECT `mode`, seq, payload FROM `{probe['table']}`;", + "--format", + "json-unicode", + timeout=20, + ) + break + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + (artifacts / f"audit-{node}-error.txt").write_text(str(error)) + if output is None: + raise RuntimeError("No available compute node could complete the independent CLI audit") (artifacts / "audit-rows.jsonl").write_text(output) missing = audit_rows(probe, json_lines(output)) (artifacts / "missing.json").write_text(json.dumps(missing, indent=2)) diff --git a/tests/slo/src/bridge_observer.py b/tests/slo/src/bridge_observer.py index 1a52dc8e1..f0aa3ef24 100644 --- a/tests/slo/src/bridge_observer.py +++ b/tests/slo/src/bridge_observer.py @@ -1,5 +1,6 @@ """Read the server's discovery RPC directly, independently of driver balancing.""" +import asyncio import grpc import ydb from ydb import _apis @@ -38,6 +39,14 @@ def close(self): channel.close() +def is_rpc_cancellation(task=None): + """A canceled RPC future is distinct from cancellation requested for its caller task.""" + task = task if task is not None else asyncio.current_task() + cancelling = getattr(task, "cancelling", None) + # Older Python versions cannot distinguish these cases; preserve cancellation there. + return cancelling is not None and cancelling() == 0 + + def route_observation(node_id, before, after): before, after = before or {}, after or {} key = str(node_id) diff --git a/tests/slo/src/primary_probe.py b/tests/slo/src/primary_probe.py index 1efeffbee..0590f750a 100644 --- a/tests/slo/src/primary_probe.py +++ b/tests/slo/src/primary_probe.py @@ -13,7 +13,7 @@ import ydb import ydb.aio -from bridge_observer import BridgeObserver, route_observation +from bridge_observer import BridgeObserver, is_rpc_cancellation, route_observation MODES = ("sync-table", "sync-query", "async-table", "async-query", "control-table") @@ -154,12 +154,21 @@ async def async_worker(self, mode): self.confirm(mode, seq) except ydb.Error as error: self.error(mode, error, "operation") + except asyncio.CancelledError as error: + if not is_rpc_cancellation(): + raise + self.error(mode, error, "rpc_cancelled") + self.emit("rpc_cancelled", mode, task_cancel_requests=0) finally: if session is not None and session.session_id: try: await session.delete(settings=ydb.BaseRequestSettings().with_timeout(5)) except ydb.Error as error: self.error(mode, error, "cleanup") + except asyncio.CancelledError as error: + if not is_rpc_cancellation(): + raise + self.error(mode, error, "cleanup_rpc_cancelled") await asyncio.sleep(1) except BaseException as error: self.fail(mode, error) diff --git a/tests/test_discovery_detect_local_dc.py b/tests/test_discovery_detect_local_dc.py index e36e301d1..1d00b15a9 100644 --- a/tests/test_discovery_detect_local_dc.py +++ b/tests/test_discovery_detect_local_dc.py @@ -344,6 +344,17 @@ def marker(second, text): for mode in MODES: for second in (1090, 1091, 1092): events.append({"event": "operation", "mode": mode, "ts": base + second, "ok": True}) + events.append( + { + "event": "session_created", + "mode": mode, + "ts": base + second, + "stable": True, + "both_piles_visible": True, + "primary": "pile-1", + "pile": "pile-1", + } + ) return "\n".join(log), events, base + 1100 From 4d4117954ecbb3893e6cd24b19ff56c3ee926404 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Thu, 12 Nov 2026 12:48:01 +0300 Subject: [PATCH 5/6] Preserve CLI audit rows around terminal control sequences --- tests/slo/bridge/checks.py | 17 +++++++ tests/slo/bridge/verify.py | 60 ++++++++++--------------- tests/test_discovery_detect_local_dc.py | 14 ++++++ 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/tests/slo/bridge/checks.py b/tests/slo/bridge/checks.py index c38be7c85..4518b5000 100644 --- a/tests/slo/bridge/checks.py +++ b/tests/slo/bridge/checks.py @@ -1,11 +1,28 @@ """Acceptance checks for independently observed live Bridge routing and data.""" from datetime import datetime +import json import re MODES = ("sync-table", "sync-query", "async-table", "async-query") +def json_lines(text, *, strict=False): + rows = [] + for line in re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text).splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + if not isinstance(row, dict): + raise ValueError("Expected a JSON object") + rows.append(row) + except (json.JSONDecodeError, ValueError): + if strict: + raise + return rows + + def stamp(value): return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() diff --git a/tests/slo/bridge/verify.py b/tests/slo/bridge/verify.py index 972a5ac9e..2209f438c 100644 --- a/tests/slo/bridge/verify.py +++ b/tests/slo/bridge/verify.py @@ -9,7 +9,7 @@ import urllib.parse import urllib.request -from checks import MODES, audit_rows, check_node_events, check_routing +from checks import MODES, audit_rows, check_node_events, check_routing, json_lines SCENARIOS = ( "01-switchover", @@ -20,22 +20,8 @@ ) -def command(*args, check=True, timeout=90): - return subprocess.run( - args, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=check, timeout=timeout - ).stdout - - -def json_lines(text): - rows = [] - for line in text.splitlines(): - try: - row = json.loads(line) - if isinstance(row, dict): - rows.append(row) - except json.JSONDecodeError: - continue - return rows +def command(*args, check=True, timeout=90, stderr=subprocess.STDOUT): + return subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=stderr, check=check, timeout=timeout).stdout def main(): @@ -103,30 +89,32 @@ def main(): output = None for node in ("ydb-database-1", "ydb-database-2"): try: - output = command( - "docker", - "exec", - node, - "/opt/ydb.tech/ydb/bin/ydb", - "-e", - "grpc://localhost:2136", - "-d", - "/Root/testdb", - "--no-discovery", - "sql", - "-s", - f"SELECT `mode`, seq, payload FROM `{probe['table']}`;", - "--format", - "json-unicode", - timeout=20, - ) + with (artifacts / f"audit-{node}-stderr.log").open("w") as stderr: + output = command( + "docker", + "exec", + node, + "/opt/ydb.tech/ydb/bin/ydb", + "-e", + "grpc://localhost:2136", + "-d", + "/Root/testdb", + "--no-discovery", + "sql", + "-s", + f"SELECT `mode`, seq, payload FROM `{probe['table']}`;", + "--format", + "json-unicode", + timeout=20, + stderr=stderr, + ) break except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: (artifacts / f"audit-{node}-error.txt").write_text(str(error)) if output is None: raise RuntimeError("No available compute node could complete the independent CLI audit") (artifacts / "audit-rows.jsonl").write_text(output) - missing = audit_rows(probe, json_lines(output)) + missing = audit_rows(probe, json_lines(output, strict=True)) (artifacts / "missing.json").write_text(json.dumps(missing, indent=2)) result["audit"] = { "confirmed": sum(len(item["confirmed"]) for item in probe["modes"].values()), @@ -185,7 +173,7 @@ def metric(expression): result["errors"].append(repr(error)) result["passed"] = not result["errors"] (artifacts / "result.json").write_text(json.dumps(result, indent=2)) - print(json.dumps(result, indent=2)) + print(json.dumps({key: value for key, value in result.items() if key != "probe"}, indent=2)) return 0 if result["passed"] else 1 diff --git a/tests/test_discovery_detect_local_dc.py b/tests/test_discovery_detect_local_dc.py index 1d00b15a9..ad5b2707d 100644 --- a/tests/test_discovery_detect_local_dc.py +++ b/tests/test_discovery_detect_local_dc.py @@ -395,6 +395,20 @@ def test_live_primary_slo_independent_audit_and_docker_evidence(): assert check_node_events(log, [stop, {"time": base + 250, "Action": "start"}]) +def test_live_primary_slo_audit_preserves_first_row_after_cli_terminal_controls(): + import json + import pytest + from tests.slo.bridge.checks import audit_rows, json_lines + + probe = {"table": "primary_probe_abc", "modes": {"async-query": {"confirmed": [1, 2]}}} + rows = [{"mode": "async-query", "seq": seq, "payload": f"primary_probe_abc:async-query:{seq}"} for seq in (1, 2)] + output = "\r\x1b[K" + "\n".join(json.dumps(row) for row in rows) + "\r\x1b[K" + assert not audit_rows(probe, json_lines(output, strict=True)) + assert audit_rows(probe, json_lines(json.dumps(rows[1]), strict=True)) == [{"mode": "async-query", "seq": 1}] + with pytest.raises(ValueError): + json_lines(output + "\nunexpected CLI output", strict=True) + + def test_live_primary_slo_oracle_and_cli_flag(): from tests.slo.src.bridge_observer import route_observation from tests.slo.src.options import get_root_parser From 97bdda5c09cb2e652f707ba40df09e312784c3cd Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Thu, 12 Nov 2026 13:27:13 +0300 Subject: [PATCH 6/6] Run the standard Bridge SLO with primary pile preference --- .github/workflows/slo-2dc.yml | 27 +-- tests/aio/test_discovery_detect_local_dc.py | 26 --- tests/slo/README.md | 49 ++--- tests/slo/bridge/checks.py | 173 ---------------- tests/slo/bridge/prepare.py | 22 +- tests/slo/bridge/switchover.sh | 58 ------ tests/slo/bridge/verify.py | 91 ++------ tests/slo/src/bridge_observer.py | 70 ------- tests/slo/src/options.py | 6 + tests/slo/src/primary_probe.py | 218 -------------------- tests/slo/src/root_runner.py | 15 +- tests/test_discovery_detect_local_dc.py | 153 ++------------ 12 files changed, 72 insertions(+), 836 deletions(-) delete mode 100644 tests/slo/bridge/checks.py delete mode 100644 tests/slo/bridge/switchover.sh delete mode 100644 tests/slo/src/bridge_observer.py delete mode 100644 tests/slo/src/primary_probe.py diff --git a/.github/workflows/slo-2dc.yml b/.github/workflows/slo-2dc.yml index f0e50aef7..a8084e517 100644 --- a/.github/workflows/slo-2dc.yml +++ b/.github/workflows/slo-2dc.yml @@ -4,7 +4,6 @@ on: pull_request: paths: - .github/workflows/slo-2dc.yml - - tests/slo/bridge/** - tests/slo/** - ydb/driver.py - ydb/resolver.py @@ -24,7 +23,7 @@ permissions: jobs: bridge: runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 30 concurrency: group: hosted-2dc-${{ github.ref }} cancel-in-progress: true @@ -68,32 +67,14 @@ jobs: python tests/slo/bridge/prepare.py .slo-action "$SERVER_IMAGE" - name: Build current Python SDK workload run: docker build -f tests/slo/Dockerfile -t py-slo-bridge:current . - - name: Record node events and stop early on a fatal probe or chaos error - run: | - docker events --filter type=container --filter label=ydb.node.type=database \ - --filter event=stop --filter event=start --format '{{json .}}' \ - > bridge-artifacts/docker-events.jsonl 2>&1 & - ( - while sleep 5; do - native=$(docker inspect --format '{{.State.Status}}' ydb-workload-current 2>/dev/null || true) - [ "$native" != exited ] || exit 0 - for container in ydb-bridge-probe ydb-chaos-monkey; do - state=$(docker inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$container" 2>/dev/null || true) - case "$state" in - exited:0|'') ;; - exited:*) docker kill ydb-workload-current >/dev/null 2>&1 || true; exit 0 ;; - esac - done - done - ) > bridge-artifacts/watchdog.log 2>&1 & - name: Run all five Bridge scenarios id: slo uses: ./.slo-action/init - timeout-minutes: 50 + timeout-minutes: 20 with: github_token: ${{ secrets.GITHUB_TOKEN }} workload_name: sync-query - workload_duration: '1500' + workload_duration: '600' workload_current_ref: current workload_current_image: py-slo-bridge:current workload_current_command: >- @@ -103,7 +84,7 @@ jobs: bridge_mode: true disable_compose_profiles: extra-nodes fail_on_workload_error: true - - name: Verify scenario coverage, SDK results and data + - name: Verify scenario coverage and workload recovery if: always() timeout-minutes: 3 env: diff --git a/tests/aio/test_discovery_detect_local_dc.py b/tests/aio/test_discovery_detect_local_dc.py index b71a587ca..ca89c1a28 100644 --- a/tests/aio/test_discovery_detect_local_dc.py +++ b/tests/aio/test_discovery_detect_local_dc.py @@ -318,29 +318,3 @@ async def test_primary_discovery_cancellation_closes_candidate(monkeypatch): await discovery.execute_discovery() assert len(PrimaryConnection.created) == 1 assert PrimaryConnection.created[0].closed - - -@pytest.mark.asyncio -async def test_live_slo_distinguishes_rpc_future_and_caller_task_cancellation(): - from tests.slo.src.bridge_observer import is_rpc_cancellation - - # Canceling an awaited future does not request cancellation of its caller task. - future = asyncio.get_running_loop().create_future() - future.cancel() - try: - await future - except asyncio.CancelledError: - assert is_rpc_cancellation() == hasattr(asyncio.current_task(), "cancelling") - - async def caller(): - try: - await asyncio.sleep(60) - except asyncio.CancelledError: - assert not is_rpc_cancellation() - raise - - task = asyncio.create_task(caller()) - await asyncio.sleep(0) - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task diff --git a/tests/slo/README.md b/tests/slo/README.md index a72eb137f..6821ee483 100644 --- a/tests/slo/README.md +++ b/tests/slo/README.md @@ -1,39 +1,20 @@ # SLO workload -## GitHub-hosted PRIMARY pile qualification - -`.github/workflows/slo-2dc.yml` runs the pinned SLO Action v2 on `ubuntu-24.04`, -using two storage and two database nodes in a real Bridge cluster. It is a ready-made -Bridge topology with `erasure=none` and SectorMap disks, not the config-v1 migration procedure. -The image is pinned by digest and its OCI metadata is retained with the results. - -The 25-minute native Query workload enables `--prefer-primary-pile`. A separate probe -keeps one driver per sync/async Table/Query mode, creates fresh sessions, writes and reads -unique transactional rows, and runs an additional disabled-option control driver. -An independent raw discovery RPC maps server-reported session node IDs to pile roles; -the SDK's cache or priority implementation is not used as the oracle. - -The first scenario stops only PRIMARY compute for 45 seconds to exercise synchronized -fallback. It then holds A, B and A as PRIMARY for 160 seconds each, without node restarts -between these observation windows. Default discovery intervals are retained. In each -window's last 30 seconds every enabled mode must create at least three sessions on PRIMARY -and confirm at least three writes; the disabled control must use both piles. - -All five upstream switchover/takedown/failover scenarios run. Every enabled mode must -resume data operations after every recovery and continue near the end. A separate CLI -query audits every acknowledged row. Transient operation errors are recorded and allowed; -wrong-pile routing after stabilization, stalled modes, lost/corrupt data, OOM, process -restarts or incomplete chaos fail the job. The oracle, workload, node events and audit -are retained in the `python-hosted-2dc` artifact for 14 days. - -Async gRPC may surface a canceled RPC future as `asyncio.CancelledError`. The probe -records it as an operation error only when no cancellation was requested for its -caller task. Caller-task cancellation is preserved. The workload image uses Python 3.11; -on runtimes without cancellation counters the probe conservatively propagates cancellation. - -The runner adapter bounds resources and cold-start readiness time, restricts bootstrap -DNS aliases to the two active compute nodes, and extends -the first scenario's observation windows; the action lifecycle remains at the pinned revision. +## GitHub-hosted Bridge SLO + +`.github/workflows/slo-2dc.yml` runs the standard five Bridge scenarios from the +pinned SLO Action v2 on `ubuntu-24.04`. The ordinary `sync-query` workload runs for +10 minutes with `--prefer-primary-pile` enabled. + +The runner uses two storage and two database nodes, `erasure=none` and SectorMap disks. +The adapter bounds resources and cold-start readiness time, restricts bootstrap DNS +to active compute nodes, and sets a 30-second gap between fault scenarios to fit the +10-minute workload. The upstream scenario scripts and SDK discovery intervals are unchanged. + +The job requires all five scenarios to complete, successful recent reads and writes, +and no workload failure, OOM or unexpected container restart. Transient operation +errors are recorded. Server image metadata, workload and chaos logs, final container +states and counters are retained in the `python-hosted-2dc` artifact for 14 days. SLO is the type of test where app based on ydb-sdk is tested against falling YDB cluster nodes, tablets, network (that is possible situations for distributed DBs with hundreds of nodes) diff --git a/tests/slo/bridge/checks.py b/tests/slo/bridge/checks.py deleted file mode 100644 index 4518b5000..000000000 --- a/tests/slo/bridge/checks.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Acceptance checks for independently observed live Bridge routing and data.""" - -from datetime import datetime -import json -import re - -MODES = ("sync-table", "sync-query", "async-table", "async-query") - - -def json_lines(text, *, strict=False): - rows = [] - for line in re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text).splitlines(): - if not line.strip(): - continue - try: - row = json.loads(line) - if not isinstance(row, dict): - raise ValueError("Expected a JSON object") - rows.append(row) - except (json.JSONDecodeError, ValueError): - if strict: - raise - return rows - - -def stamp(value): - return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() - - -def check_routing(log, events, stopped_at): - errors, windows, beginnings = [], [], {} - pattern = r"\[([^\]]+)\] PRIMARY_WINDOW_(BEGIN|END) (\w+) (\S+)" - for when, kind, name, pile in re.findall(pattern, log): - if kind == "BEGIN": - beginnings[name] = (stamp(when), pile) - elif name in beginnings: - start, initial_pile = beginnings.pop(name) - windows.append({"name": name, "start": start, "end": stamp(when), "pile": pile}) - if pile != initial_pile or stamp(when) - start < 150: - errors.append(f"Invalid PRIMARY observation window: {name}") - if [w["name"] for w in windows] != ["initial", "switched", "restored"] or beginnings: - errors.append("Missing the three controlled PRIMARY observation windows") - elif not (windows[0]["pile"] == windows[2]["pile"] != windows[1]["pile"]): - errors.append("PRIMARY did not switch to the other pile and back") - details = {"primary_windows": [], "recovery_windows": [], "fallback": {}} - for window in windows: - end, primary = window["end"], window["pile"] - late = [row for row in events if end - 30 <= row.get("ts", 0) <= end] - routes = [ - row - for row in late - if row.get("event") == "session_created" - and row.get("stable") - and row.get("both_piles_visible") - and row.get("primary") == primary - ] - observation = {**window, "modes": {}} - for mode in MODES: - samples = [row for row in routes if row.get("mode") == mode] - writes = sum(row.get("event") == "operation" and row.get("ok") and row.get("mode") == mode for row in late) - wrong = [row for row in samples if row.get("pile") != primary] - observation["modes"][mode] = {"sessions": len(samples), "wrong_pile": len(wrong), "writes": writes} - if len(samples) < 3 or writes < 3 or wrong: - errors.append(f"PRIMARY routing or data progress failed for {mode} in {window['name']}") - control = {row.get("pile") for row in routes if row.get("mode") == "control-table"} - observation["control_piles"] = sorted(pile for pile in control if pile) - if control != {"pile-1", "pile-2"}: - errors.append(f"Disabled-option control did not use both piles in {window['name']}") - details["primary_windows"].append(observation) - fallback = re.findall(r"\[([^\]]+)\] PRIMARY_FALLBACK_(BEGIN|END) (\S+)", log) - if len(fallback) != 2 or [item[1] for item in fallback] != ["BEGIN", "END"] or fallback[0][2] != fallback[1][2]: - errors.append("Missing controlled primary-compute outage") - else: - start, end = stamp(fallback[0][0]), stamp(fallback[1][0]) - for mode in MODES: - count = sum( - row.get("event") == "session_created" - and row.get("mode") == mode - and start + 2 <= row.get("ts", 0) <= end - and row.get("stable") - and row.get("fallback") - and row.get("primary") == fallback[0][2] - for row in events - ) - details["fallback"][mode] = count - if not count: - errors.append(f"No synchronized fallback observed for {mode} while PRIMARY compute was stopped") - recoveries = re.findall(r"\[([^\]]+)\] Recovered: ([^\n]+)", log) - if len(recoveries) != 5: - errors.append(f"Expected five completed chaos recoveries, found {len(recoveries)}") - for when, name in recoveries: - start = stamp(when) - counts = { - mode: sum( - row.get("event") == "operation" - and row.get("ok") - and row.get("mode") == mode - and start <= row.get("ts", 0) <= start + 90 - for row in events - ) - for mode in MODES - } - details["recovery_windows"].append({"name": name, "writes": counts}) - if any(count < 3 for count in counts.values()): - errors.append(f"Some probe modes did not recover after {name}") - details["final_writes"] = { - mode: sum( - row.get("event") == "operation" - and row.get("ok") - and row.get("mode") == mode - and stopped_at - 60 <= row.get("ts", 0) <= stopped_at - for row in events - ) - for mode in MODES - } - if any(count < 3 for count in details["final_writes"].values()): - errors.append("Some probe modes stopped making progress at the end of the run") - details["final_routing"] = {} - if windows: - primary = windows[0]["pile"] - for mode in MODES: - samples = [ - row - for row in events - if row.get("event") == "session_created" - and row.get("mode") == mode - and stopped_at - 30 <= row.get("ts", 0) <= stopped_at - and row.get("stable") - and row.get("both_piles_visible") - and row.get("primary") == primary - ] - wrong = sum(row.get("pile") != primary for row in samples) - details["final_routing"][mode] = {"sessions": len(samples), "wrong_pile": wrong} - if len(samples) < 3 or wrong: - errors.append(f"PRIMARY routing did not recover at the end for {mode}") - return details, errors - - -def audit_rows(probe, rows): - stored = {(row["mode"], int(row["seq"])): row["payload"] for row in rows} - return [ - {"mode": mode, "seq": seq} - for mode, result in probe["modes"].items() - for seq in result["confirmed"] - if stored.get((mode, seq)) != f"{probe['table']}:{mode}:{seq}" - ] - - -def check_node_events(log, events): - windows = re.findall(r"\[([^\]]+)\] PRIMARY_WINDOW_(BEGIN|END) (\w+) (\S+)", log) - if len(windows) != 6: - return ["Missing node-event observation boundaries"] - first, last = stamp(windows[0][0]), stamp(windows[-1][0]) - errors = [] - for event in events: - when = event["timeNano"] / 1e9 if event.get("timeNano") else event["time"] - if first <= when <= last: - errors.append("A database process changed during the role-only observation windows") - break - fallback = re.findall(r"\[([^\]]+)\] PRIMARY_FALLBACK_(BEGIN|END) (\S+)", log) - if len(fallback) != 2: - return errors + ["Missing controlled outage node boundaries"] - start, end, pile = stamp(fallback[0][0]), stamp(fallback[1][0]), fallback[0][2] - stops = [ - event - for event in events - if (event.get("Action") or event.get("status")) == "stop" - and event.get("Actor", {}).get("Attributes", {}).get("ydb.node.location.pile") == pile - and start <= (event["timeNano"] / 1e9 if event.get("timeNano") else event["time"]) <= end - ] - if not stops: - errors.append("No Docker evidence that PRIMARY compute actually stopped") - return errors diff --git a/tests/slo/bridge/prepare.py b/tests/slo/bridge/prepare.py index 92e01bd56..b0b14bdbf 100644 --- a/tests/slo/bridge/prepare.py +++ b/tests/slo/bridge/prepare.py @@ -1,10 +1,8 @@ """Configure a pinned Action checkout for a bounded GitHub-hosted run.""" import argparse -import copy import json import subprocess -import shutil from pathlib import Path @@ -43,6 +41,11 @@ def prepare(action, image): "ydb:172.28.0.11", "ydb:172.28.0.12", ] + chaos = config["services"]["chaos-monkey"] + environment = chaos["environment"] + if isinstance(environment, list): + environment = {item.partition("=")[0]: item.partition("=")[2] if "=" in item else None for item in environment} + chaos["environment"] = {**environment, "CHAOS_SCENARIO_DELAY": "30"} for filename in ("config.yaml", "config.bridge.yaml"): path = deploy / "ydb/rootfs/opt/ydb.tech/ydbd/cfg" / filename text = path.read_text() @@ -58,21 +61,6 @@ def prepare(action, image): if text.count("local max_attempts=30") != 2: raise RuntimeError("Unexpected readiness implementation") readiness.write_text(text.replace("local max_attempts=30", "local max_attempts=150", 1)) - scenario = deploy / "chaos/rootfs/opt/ydb.tech/chaos/bridge-scenarios/01-switchover.sh" - shutil.copyfile(Path(__file__).with_name("switchover.sh"), scenario) - scenario.chmod(0o755) - probe = copy.deepcopy(config["services"]["workload-current"]) - probe.update( - container_name="ydb-bridge-probe", entrypoint=["python", "/src/tests/slo/src/primary_probe.py"], command=[] - ) - probe.pop("deploy", None) - probe["mem_limit"] = "768m" - probe["restart"] = "no" - environment = probe.get("environment", {}) - if isinstance(environment, list): - environment = {item.partition("=")[0]: item.partition("=")[2] if "=" in item else None for item in environment} - probe["environment"] = {**environment, "ORACLE_ENDPOINTS": "ydb-storage-1:2136,ydb-storage-2:2136"} - config["services"]["bridge-probe"] = probe compose.write_text(json.dumps(config, indent=2) + "\n") subprocess.run(["docker", "compose", "-f", str(compose), "config", "--quiet"], check=True) diff --git a/tests/slo/bridge/switchover.sh b/tests/slo/bridge/switchover.sh deleted file mode 100644 index 626dcd16b..000000000 --- a/tests/slo/bridge/switchover.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -set -e -. /opt/ydb.tech/scripts/chaos/libchaos.sh - -recovery_timeout="${BRIDGE_RECOVERY_TIMEOUT:-180}" -original_primary=$(get_bridge_pile_by_state PRIMARY) -new_primary=$(get_bridge_pile_by_state SYNCHRONIZED) -compute=$(get_bridge_pile_nodes "$original_primary" database) -fault_active=false - -cleanup() { - set +e - docker start "$compute" >/dev/null - restore_bridge_primary "$original_primary" "$recovery_timeout" - if [ "$fault_active" = true ]; then - chaos_recover "bridge-switchover" "$new_primary" "Original PRIMARY restored" - fi -} - -[ -n "$original_primary" ] && [ -n "$new_primary" ] && [ -n "$compute" ] -BRIDGE_CLI_PILE="$original_primary" -install_bridge_cleanup_traps - -deadline=$(($(date +%s) + 120)) -until docker exec ydb-bridge-probe test -f /tmp/bridge-probe-ready; do - [ "$(date +%s)" -lt "$deadline" ] || exit 1 - sleep 2 -done - -log "PRIMARY_FALLBACK_BEGIN $original_primary" -docker stop -t 0 "$compute" -sleep 45 -log "PRIMARY_FALLBACK_END $original_primary" -docker start "$compute" >/dev/null -wait_container_healthy "$compute" "$recovery_timeout" - -log "PRIMARY_WINDOW_BEGIN initial $original_primary" -sleep 160 -log "PRIMARY_WINDOW_END initial $original_primary" - -chaos_inject "bridge-switchover" "$new_primary" -fault_active=true -bridge_cli switchover --new-primary "$new_primary" -wait_bridge_pile_state "$new_primary" PRIMARY "$recovery_timeout" -BRIDGE_CLI_PILE="$new_primary" -log "PRIMARY_WINDOW_BEGIN switched $new_primary" -sleep 160 -log "PRIMARY_WINDOW_END switched $new_primary" - -restore_bridge_primary "$original_primary" "$recovery_timeout" -BRIDGE_CLI_PILE="$original_primary" -wait_bridge_pile_state "$new_primary" SYNCHRONIZED "$recovery_timeout" -log "PRIMARY_WINDOW_BEGIN restored $original_primary" -sleep 160 -log "PRIMARY_WINDOW_END restored $original_primary" -chaos_recover "bridge-switchover" "$new_primary" "Original PRIMARY restored" -fault_active=false -clear_bridge_cleanup_traps diff --git a/tests/slo/bridge/verify.py b/tests/slo/bridge/verify.py index 2209f438c..14257cb48 100644 --- a/tests/slo/bridge/verify.py +++ b/tests/slo/bridge/verify.py @@ -1,16 +1,13 @@ -"""Verify live role-aware routing, recovery and independently audited data.""" +"""Verify completion and recovery of the GitHub-hosted Bridge workload.""" import json import math import os from pathlib import Path -import re import subprocess import urllib.parse import urllib.request -from checks import MODES, audit_rows, check_node_events, check_routing, json_lines - SCENARIOS = ( "01-switchover", "02-takedown-synchronized", @@ -24,6 +21,18 @@ def command(*args, check=True, timeout=90, stderr=subprocess.STDOUT): return subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=stderr, check=check, timeout=timeout).stdout +def check_chaos(log, state, restarts): + errors = [] + if state["Status"] != "exited" or state["ExitCode"] != 0 or restarts or "Chaos Monkey finished" not in log: + errors.append("Chaos sequence did not finish without a process restart") + for scenario in SCENARIOS: + if f"Running scenario: {scenario}" not in log: + errors.append(f"Scenario not executed: {scenario}") + if log.count("] Recovered: ") != len(SCENARIOS): + errors.append("Expected five completed chaos recoveries") + return errors + + def main(): artifacts = Path("bridge-artifacts") artifacts.mkdir(exist_ok=True) @@ -55,73 +64,8 @@ def main(): chaos_info = json.loads(command("docker", "inspect", "ydb-chaos-monkey"))[0] chaos = chaos_info["State"] log = (artifacts / "ydb-chaos-monkey.log").read_text() - if ( - chaos["Status"] != "exited" - or chaos["ExitCode"] != 0 - or chaos_info["RestartCount"] - or "Chaos Monkey finished" not in log - ): - result["errors"].append("Chaos sequence did not finish without a process restart") - for scenario in SCENARIOS: - if f"Running scenario: {scenario}" not in log: - result["errors"].append(f"Scenario not executed: {scenario}") - state = json.loads(command("docker", "inspect", "ydb-bridge-probe"))[0] - if state["State"]["Running"]: - command("docker", "exec", "ydb-bridge-probe", "touch", "/tmp/bridge-probe-stop") - command("docker", "wait", "ydb-bridge-probe", timeout=150) - probe_log = command("docker", "logs", "ydb-bridge-probe", check=False) - (artifacts / "ydb-bridge-probe.log").write_text(probe_log) - command("docker", "cp", "ydb-bridge-probe:/tmp/bridge-probe.json", str(artifacts / "probe.json")) - probe = json.loads((artifacts / "probe.json").read_text()) - result["probe"] = probe - if not probe.get("passed") or state["RestartCount"]: - result["errors"].append("Probe failed or its process was restarted") - events = json_lines(probe_log) - for mode in (*MODES, "control-table"): - if sum(row.get("event") == "driver_started" and row.get("mode") == mode for row in events) != 1: - result["errors"].append(f"Expected one long-lived driver for {mode}") - routing, errors = check_routing(log, events, probe["stopped_at"]) - result["routing"] = routing - result["errors"].extend(errors) - result["errors"].extend(check_node_events(log, json_lines((artifacts / "docker-events.jsonl").read_text()))) - if not re.fullmatch(r"primary_probe_[a-f0-9]+", probe["table"]): - raise ValueError("Unexpected audit table name") - output = None - for node in ("ydb-database-1", "ydb-database-2"): - try: - with (artifacts / f"audit-{node}-stderr.log").open("w") as stderr: - output = command( - "docker", - "exec", - node, - "/opt/ydb.tech/ydb/bin/ydb", - "-e", - "grpc://localhost:2136", - "-d", - "/Root/testdb", - "--no-discovery", - "sql", - "-s", - f"SELECT `mode`, seq, payload FROM `{probe['table']}`;", - "--format", - "json-unicode", - timeout=20, - stderr=stderr, - ) - break - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: - (artifacts / f"audit-{node}-error.txt").write_text(str(error)) - if output is None: - raise RuntimeError("No available compute node could complete the independent CLI audit") - (artifacts / "audit-rows.jsonl").write_text(output) - missing = audit_rows(probe, json_lines(output, strict=True)) - (artifacts / "missing.json").write_text(json.dumps(missing, indent=2)) - result["audit"] = { - "confirmed": sum(len(item["confirmed"]) for item in probe["modes"].values()), - "missing": len(missing), - } - if missing: - result["errors"].append("Acknowledged probe rows were lost or corrupted") + result["errors"].extend(check_chaos(log, chaos, chaos_info["RestartCount"])) + result["completed_scenarios"] = len(SCENARIOS) if not result["errors"] else None ip = command( "docker", "inspect", @@ -156,7 +100,6 @@ def metric(expression): "ydb-database-1", "ydb-database-2", "ydb-workload-current", - "ydb-bridge-probe", ) ) result["container_states"] = { @@ -165,7 +108,7 @@ def metric(expression): if any(node["State"]["OOMKilled"] or node["RestartCount"] for node in containers): result["errors"].append("A container was OOM killed or unexpectedly restarted") for node in containers: - if node["Name"] in ("/ydb-workload-current", "/ydb-bridge-probe") and ( + if node["Name"] == "/ydb-workload-current" and ( node["State"]["Status"] != "exited" or node["State"]["ExitCode"] != 0 ): result["errors"].append(f"Workload process did not exit successfully: {node['Name']}") @@ -173,7 +116,7 @@ def metric(expression): result["errors"].append(repr(error)) result["passed"] = not result["errors"] (artifacts / "result.json").write_text(json.dumps(result, indent=2)) - print(json.dumps({key: value for key, value in result.items() if key != "probe"}, indent=2)) + print(json.dumps(result, indent=2)) return 0 if result["passed"] else 1 diff --git a/tests/slo/src/bridge_observer.py b/tests/slo/src/bridge_observer.py deleted file mode 100644 index f0aa3ef24..000000000 --- a/tests/slo/src/bridge_observer.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Read the server's discovery RPC directly, independently of driver balancing.""" - -import asyncio -import grpc -import ydb -from ydb import _apis - - -class BridgeObserver: - def __init__(self, endpoints, database): - self.database = database - self.channels = [grpc.insecure_channel(endpoint) for endpoint in endpoints] - self.stubs = [_apis.DiscoveryService.Stub(channel) for channel in self.channels] - - def snapshot(self): - for stub in self.stubs: - try: - response = stub.ListEndpoints( - _apis.ydb_discovery.ListEndpointsRequest(database=self.database), timeout=2 - ) - if response.operation.status != ydb.StatusCode.SUCCESS: - continue - result = _apis.ydb_discovery.ListEndpointsResult() - if not response.operation.result.Unpack(result): - continue - states = {pile.pile_name: pile.state for pile in result.pile_states} - primary = next((name for name, state in states.items() if state == 1), None) - return { - "states": states, - "primary": primary, - "node_piles": {str(ep.node_id): ep.bridge_pile_name for ep in result.endpoints if not ep.ssl}, - } - except grpc.RpcError: - continue - return None - - def close(self): - for channel in self.channels: - channel.close() - - -def is_rpc_cancellation(task=None): - """A canceled RPC future is distinct from cancellation requested for its caller task.""" - task = task if task is not None else asyncio.current_task() - cancelling = getattr(task, "cancelling", None) - # Older Python versions cannot distinguish these cases; preserve cancellation there. - return cancelling is not None and cancelling() == 0 - - -def route_observation(node_id, before, after): - before, after = before or {}, after or {} - key = str(node_id) - pile = after.get("node_piles", {}).get(key) - stable = bool( - before.get("primary") - and before.get("primary") == after.get("primary") - and before.get("states") == after.get("states") - and pile - and before.get("node_piles", {}).get(key) == pile - ) - return { - "node_id": node_id, - "pile": pile, - "primary": after.get("primary"), - "stable": stable, - "both_piles_visible": {"pile-1", "pile-2"} <= set(after.get("node_piles", {}).values()), - "fallback": stable and after["states"].get(pile) == 3 and pile != after["primary"], - "before": before, - "after": after, - } diff --git a/tests/slo/src/options.py b/tests/slo/src/options.py index 08e33d3db..9f289df45 100644 --- a/tests/slo/src/options.py +++ b/tests/slo/src/options.py @@ -1,4 +1,10 @@ import argparse +import ydb + + +def get_driver_config(args): + options = {"prefer_primary_pile": True} if getattr(args, "prefer_primary_pile", False) else {} + return ydb.DriverConfig(args.endpoint, database=args.db, grpc_keep_alive_timeout=5000, **options) def add_common_options(parser): diff --git a/tests/slo/src/primary_probe.py b/tests/slo/src/primary_probe.py deleted file mode 100644 index 0590f750a..000000000 --- a/tests/slo/src/primary_probe.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Exercise fresh-session routing and acknowledged data on long-lived drivers.""" - -import asyncio -import json -import os -from pathlib import Path -import threading -import time -import traceback -from urllib.parse import parse_qs, urlparse -import uuid - -import ydb -import ydb.aio - -from bridge_observer import BridgeObserver, is_rpc_cancellation, route_observation - -MODES = ("sync-table", "sync-query", "async-table", "async-query", "control-table") - - -class Probe: - def __init__(self): - self.table = "primary_probe_" + uuid.uuid4().hex - self.stop = threading.Event() - self.lock = threading.Lock() - self.stats = {mode: {"successes": 0, "errors": [], "confirmed": [], "driver_starts": 0} for mode in MODES} - self.fatal = [] - self.observer = BridgeObserver(os.environ["ORACLE_ENDPOINTS"].split(","), os.environ["YDB_DATABASE"]) - - def emit(self, event, mode, **values): - with self.lock: - print(json.dumps({"event": event, "mode": mode, "ts": time.time(), **values}), flush=True) - - def error(self, mode, error, stage): - with self.lock: - self.stats[mode]["errors"].append({"stage": stage, "error": repr(error)}) - self.emit("error", mode, error=repr(error), stage=stage) - - def fail(self, mode, error): - with self.lock: - self.fatal.append({"mode": mode, "error": repr(error)}) - traceback.print_exc() - self.stop.set() - - def config(self, mode): - return ydb.DriverConfig( - os.environ["YDB_ENDPOINT"], - database=os.environ["YDB_DATABASE"], - credentials=ydb.AnonymousCredentials(), - prefer_primary_pile=mode != "control-table", - ) - - def sql(self, mode, seq): - payload = f"{self.table}:{mode}:{seq}" - return ( - f"UPSERT INTO `{self.table}` (`mode`, seq, payload) VALUES " - f'(CAST("{mode}" AS Utf8), CAST({seq} AS Uint64), CAST("{payload}" AS Utf8)); ' - f'SELECT payload FROM `{self.table}` WHERE `mode` = CAST("{mode}" AS Utf8) ' - f"AND seq = CAST({seq} AS Uint64);" - ) - - def validate(self, mode, seq, rows): - if len(rows) != 1 or len(rows[0].rows) != 1 or rows[0].rows[0].payload != f"{self.table}:{mode}:{seq}": - raise AssertionError(f"Incorrect read-after-write for {mode}/{seq}") - - def confirm(self, mode, seq): - with self.lock: - self.stats[mode]["successes"] += 1 - self.stats[mode]["confirmed"].append(seq) - if all(self.stats[name]["successes"] for name in MODES): - Path("/tmp/bridge-probe-ready").touch() - self.emit("operation", mode, seq=seq, ok=True) - - def started(self, mode): - with self.lock: - self.stats[mode]["driver_starts"] += 1 - self.emit("driver_started", mode, prefer_primary_pile=mode != "control-table", sdk=ydb.__version__) - - def session_created(self, mode, session, before, after): - if mode.endswith("query"): - node_id = session.node_id - else: - node_id = int(parse_qs(urlparse(session.session_id).query)["node_id"][0]) - self.emit("session_created", mode, **route_observation(node_id, before, after)) - - def sync_worker(self, mode): - try: - with ydb.Driver(self.config(mode)) as driver: - driver.wait(timeout=60) - self.started(mode) - seq = 0 - while not self.stop.is_set(): - seq += 1 - session = None - try: - before = self.observer.snapshot() - session = ydb.QuerySession(driver) if mode.endswith("query") else driver.table_client.session() - session.create(settings=ydb.BaseRequestSettings().with_timeout(5)) - self.session_created(mode, session, before, self.observer.snapshot()) - if mode.endswith("query"): - with session.transaction(ydb.QuerySerializableReadWrite()) as tx: - with tx.execute( - self.sql(mode, seq), settings=ydb.BaseRequestSettings().with_timeout(5) - ) as stream: - self.validate(mode, seq, list(stream)) - tx.commit(settings=ydb.BaseRequestSettings().with_timeout(5)) - else: - rows = session.transaction(ydb.SerializableReadWrite()).execute( - self.sql(mode, seq), commit_tx=True, settings=ydb.BaseRequestSettings().with_timeout(5) - ) - self.validate(mode, seq, rows) - self.confirm(mode, seq) - except ydb.Error as error: - self.error(mode, error, "operation") - finally: - if session is not None and session.session_id: - try: - session.delete(settings=ydb.BaseRequestSettings().with_timeout(5)) - except ydb.Error as error: - self.error(mode, error, "cleanup") - self.stop.wait(1) - except BaseException as error: - self.fail(mode, error) - - async def async_worker(self, mode): - try: - async with ydb.aio.Driver(self.config(mode)) as driver: - await driver.wait(timeout=60) - self.started(mode) - seq = 0 - while not self.stop.is_set(): - seq += 1 - session = None - try: - before = await asyncio.to_thread(self.observer.snapshot) - session = ( - ydb.aio.QuerySession(driver) if mode.endswith("query") else driver.table_client.session() - ) - await session.create(settings=ydb.BaseRequestSettings().with_timeout(5)) - after = await asyncio.to_thread(self.observer.snapshot) - self.session_created(mode, session, before, after) - if mode.endswith("query"): - async with session.transaction(ydb.QuerySerializableReadWrite()) as tx: - async with await tx.execute( - self.sql(mode, seq), settings=ydb.BaseRequestSettings().with_timeout(5) - ) as stream: - self.validate(mode, seq, [part async for part in stream]) - await tx.commit(settings=ydb.BaseRequestSettings().with_timeout(5)) - else: - rows = await session.transaction(ydb.SerializableReadWrite()).execute( - self.sql(mode, seq), commit_tx=True, settings=ydb.BaseRequestSettings().with_timeout(5) - ) - self.validate(mode, seq, rows) - self.confirm(mode, seq) - except ydb.Error as error: - self.error(mode, error, "operation") - except asyncio.CancelledError as error: - if not is_rpc_cancellation(): - raise - self.error(mode, error, "rpc_cancelled") - self.emit("rpc_cancelled", mode, task_cancel_requests=0) - finally: - if session is not None and session.session_id: - try: - await session.delete(settings=ydb.BaseRequestSettings().with_timeout(5)) - except ydb.Error as error: - self.error(mode, error, "cleanup") - except asyncio.CancelledError as error: - if not is_rpc_cancellation(): - raise - self.error(mode, error, "cleanup_rpc_cancelled") - await asyncio.sleep(1) - except BaseException as error: - self.fail(mode, error) - - async def async_workers(self): - await asyncio.gather(self.async_worker("async-table"), self.async_worker("async-query")) - - def run(self): - with ydb.Driver(self.config("control-table")) as setup: - setup.wait(timeout=60) - with ydb.QuerySessionPool(setup) as pool: - version = pool.execute_with_retries("SELECT CAST(version() AS Utf8) AS version;")[0].rows[0].version - pool.execute_with_retries( - f"CREATE TABLE `{self.table}` (`mode` Utf8, seq Uint64, payload Utf8, PRIMARY KEY(`mode`, seq));" - ) - threads = [ - threading.Thread(target=self.sync_worker, args=(mode,), daemon=True) - for mode in ("sync-table", "sync-query", "control-table") - ] - threads.append(threading.Thread(target=lambda: asyncio.run(self.async_workers()), daemon=True)) - for thread in threads: - thread.start() - deadline = time.monotonic() + int(os.environ["WORKLOAD_DURATION"]) + 180 - while not self.stop.is_set() and time.monotonic() < deadline and not Path("/tmp/bridge-probe-stop").exists(): - time.sleep(0.5) - stopped_at = time.time() - self.stop.set() - for thread in threads: - thread.join(timeout=60) - if any(thread.is_alive() for thread in threads): - self.fatal.append({"error": "Probe worker did not finish"}) - self.observer.close() - result = { - "table": self.table, - "server_version": version, - "modes": self.stats, - "fatal": self.fatal, - "stopped_at": stopped_at, - "passed": not self.fatal and all(self.stats[mode]["successes"] for mode in MODES), - } - Path("/tmp/bridge-probe.json").write_text(json.dumps(result, indent=2)) - print(json.dumps(result), flush=True) - return 0 if result["passed"] else 1 - - -if __name__ == "__main__": - raise SystemExit(Probe().run()) diff --git a/tests/slo/src/root_runner.py b/tests/slo/src/root_runner.py index 5b7f5657e..5334a7ff8 100644 --- a/tests/slo/src/root_runner.py +++ b/tests/slo/src/root_runner.py @@ -5,6 +5,7 @@ from typing import Dict from core.metrics import WORKLOAD +from options import get_driver_config from runners.topic_runner import TopicRunner from runners.table_runner import TableRunner from runners.base import BaseRunner @@ -45,12 +46,7 @@ def run_command(self, args): def _run_sync_command(self, args, runner_instance, command): """Run command in synchronous mode""" - driver_config = ydb.DriverConfig( - args.endpoint, - database=args.db, - grpc_keep_alive_timeout=5000, - prefer_primary_pile=getattr(args, "prefer_primary_pile", False), - ) + driver_config = get_driver_config(args) with ydb.Driver(driver_config) as driver: driver.wait(timeout=300) @@ -72,12 +68,7 @@ def _run_sync_command(self, args, runner_instance, command): async def _run_async_command(self, args, runner_instance, command): """Run command in asynchronous mode""" - driver_config = ydb.DriverConfig( - args.endpoint, - database=args.db, - grpc_keep_alive_timeout=5000, - prefer_primary_pile=getattr(args, "prefer_primary_pile", False), - ) + driver_config = get_driver_config(args) async with ydb.aio.Driver(driver_config) as driver: await driver.wait(timeout=300) diff --git a/tests/test_discovery_detect_local_dc.py b/tests/test_discovery_detect_local_dc.py index ad5b2707d..77f30a79a 100644 --- a/tests/test_discovery_detect_local_dc.py +++ b/tests/test_discovery_detect_local_dc.py @@ -282,140 +282,31 @@ def test_primary_driver_keyword_and_config_override(monkeypatch): assert not seen[-1].prefer_primary_pile -def slo_observations(): - from datetime import datetime, timezone - from tests.slo.bridge.checks import MODES - - base = datetime(2026, 1, 1, tzinfo=timezone.utc).timestamp() - log, events = [], [] - - def marker(second, text): - when = datetime.fromtimestamp(base + second, tz=timezone.utc).isoformat().replace("+00:00", "Z") - log.append(f"[{when}] {text}") - - marker(0, "PRIMARY_FALLBACK_BEGIN pile-1") - marker(20, "PRIMARY_FALLBACK_END pile-1") - for mode in MODES: - events.append( - { - "event": "session_created", - "mode": mode, - "ts": base + 10, - "stable": True, - "fallback": True, - "primary": "pile-1", - "pile": "pile-2", - } - ) - for name, start, pile in (("initial", 30, "pile-1"), ("switched", 200, "pile-2"), ("restored", 370, "pile-1")): - marker(start, f"PRIMARY_WINDOW_BEGIN {name} {pile}") - marker(start + 160, f"PRIMARY_WINDOW_END {name} {pile}") - for mode in MODES: - for offset in (140, 145, 150): - events.append( - { - "event": "session_created", - "mode": mode, - "ts": base + start + offset, - "stable": True, - "both_piles_visible": True, - "primary": pile, - "pile": pile, - } - ) - events.append({"event": "operation", "mode": mode, "ts": base + start + offset, "ok": True}) - for control_pile in ("pile-1", "pile-2"): - events.append( - { - "event": "session_created", - "mode": "control-table", - "ts": base + start + 145, - "stable": True, - "both_piles_visible": True, - "primary": pile, - "pile": control_pile, - } - ) - for second in (540, 650, 760, 870, 980): - marker(second, "Recovered: scenario (restored)") - for mode in MODES: - for offset in (1, 2, 3): - events.append({"event": "operation", "mode": mode, "ts": base + second + offset, "ok": True}) - for mode in MODES: - for second in (1090, 1091, 1092): - events.append({"event": "operation", "mode": mode, "ts": base + second, "ok": True}) - events.append( - { - "event": "session_created", - "mode": mode, - "ts": base + second, - "stable": True, - "both_piles_visible": True, - "primary": "pile-1", - "pile": "pile-1", - } - ) - return "\n".join(log), events, base + 1100 - +def test_slo_primary_option_keeps_legacy_baseline_compatible(monkeypatch): + from tests.slo.src import options -def test_live_primary_slo_acceptance_and_wrong_pile_rejection(): - from tests.slo.bridge.checks import check_routing - - log, events, stopped_at = slo_observations() - assert not check_routing(log, events, stopped_at)[1] - sample = next(row for row in events if row.get("both_piles_visible") and row["mode"] == "sync-table") - sample["pile"] = "pile-2" - assert any("sync-table" in error for error in check_routing(log, events, stopped_at)[1]) + args = options.get_root_parser().parse_args( + ["table-run", "grpc://host:2136", "/Root/testdb", "--prefer-primary-pile"] + ) + assert options.get_driver_config(args).prefer_primary_pile + original = driver.DriverConfig + def legacy_config(endpoint, database, grpc_keep_alive_timeout): + return original(endpoint, database=database, grpc_keep_alive_timeout=grpc_keep_alive_timeout) -def test_live_primary_slo_requires_control_and_all_modes(): - from tests.slo.bridge.checks import check_routing + monkeypatch.setattr(options.ydb, "DriverConfig", legacy_config) + args = options.get_root_parser().parse_args(["table-run", "grpc://host:2136", "/Root/testdb"]) + assert not args.prefer_primary_pile + assert not options.get_driver_config(args).prefer_primary_pile - log, events, stopped_at = slo_observations() - events = [row for row in events if row["mode"] != "control-table"] - assert any("control" in error for error in check_routing(log, events, stopped_at)[1]) - events = [row for row in events if row["mode"] != "async-query"] - assert any( - "async-query" in error or "Some probe modes" in error for error in check_routing(log, events, stopped_at)[1] - ) +def test_bridge_slo_requires_all_completed_scenarios(): + from tests.slo.bridge.verify import SCENARIOS, check_chaos -def test_live_primary_slo_independent_audit_and_docker_evidence(): - from tests.slo.bridge.checks import audit_rows, check_node_events - - probe = {"table": "primary_probe_abc", "modes": {"sync-table": {"confirmed": [1, 2]}}} - rows = [{"mode": "sync-table", "seq": seq, "payload": f"primary_probe_abc:sync-table:{seq}"} for seq in (1, 2)] - assert not audit_rows(probe, rows) - rows[1]["payload"] = "corrupted" - assert audit_rows(probe, rows) == [{"mode": "sync-table", "seq": 2}] - log, _, stopped_at = slo_observations() - base = stopped_at - 1100 - stop = {"time": base + 5, "Action": "stop", "Actor": {"Attributes": {"ydb.node.location.pile": "pile-1"}}} - assert not check_node_events(log, [stop]) - assert check_node_events(log, [stop, {"time": base + 250, "Action": "start"}]) - - -def test_live_primary_slo_audit_preserves_first_row_after_cli_terminal_controls(): - import json - import pytest - from tests.slo.bridge.checks import audit_rows, json_lines - - probe = {"table": "primary_probe_abc", "modes": {"async-query": {"confirmed": [1, 2]}}} - rows = [{"mode": "async-query", "seq": seq, "payload": f"primary_probe_abc:async-query:{seq}"} for seq in (1, 2)] - output = "\r\x1b[K" + "\n".join(json.dumps(row) for row in rows) + "\r\x1b[K" - assert not audit_rows(probe, json_lines(output, strict=True)) - assert audit_rows(probe, json_lines(json.dumps(rows[1]), strict=True)) == [{"mode": "async-query", "seq": 1}] - with pytest.raises(ValueError): - json_lines(output + "\nunexpected CLI output", strict=True) - - -def test_live_primary_slo_oracle_and_cli_flag(): - from tests.slo.src.bridge_observer import route_observation - from tests.slo.src.options import get_root_parser - - before = {"states": {"pile-1": 1, "pile-2": 3}, "primary": "pile-1", "node_piles": {"1": "pile-1", "2": "pile-2"}} - assert route_observation(1, before, before)["stable"] - assert route_observation(2, before, before)["fallback"] - assert not route_observation(1, before, {**before, "primary": "pile-2"})["stable"] - args = get_root_parser().parse_args(["table-run", "grpc://host:2136", "/Root/testdb", "--prefer-primary-pile"]) - assert args.prefer_primary_pile + log = "\n".join(f"Running scenario: {name}\n[time] Recovered: {name}" for name in SCENARIOS) + log += "\nChaos Monkey finished" + state = {"Status": "exited", "ExitCode": 0} + assert not check_chaos(log, state, 0) + assert check_chaos(log.replace("Running scenario: 05-failover-primary", "skipped"), state, 0) + assert check_chaos(log.replace("] Recovered: 05-failover-primary", "] incomplete"), state, 0) + assert check_chaos(log, state, 1)