From 67ca9208bdbefe6545bcb3925fb29ffa8bf4ac8b Mon Sep 17 00:00:00 2001 From: mregrock <78217843+mregrock@users.noreply.github.com> Date: Thu, 5 Nov 2026 09:00:17 +0000 Subject: [PATCH 1/2] Preserve active streams during connection retirement --- CHANGELOG.md | 1 + ydb/aio/pool.py | 5 +- ydb/aio/query/pool_test.py | 24 +++ ydb/connection.py | 27 ++-- ydb/pool.py | 25 +-- ydb/query/pool_test.py | 309 ++++++++++++++++++++++++++++++++++++- 6 files changed, 364 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec59bd40..bc0d47eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +* Keep synchronous streams alive during discovery cleanup and safely close their connections on driver shutdown. * Fix topic reader `close()` hanging forever when the stream is broken with uncommitted offsets pending: the commit flush now stops as soon as the stream reports an error instead of waiting for acks that can never arrive * Fix async table session pool capacity leaking when keeping an idle session alive fails or is cancelled diff --git a/ydb/aio/pool.py b/ydb/aio/pool.py index 15027436..ab6b02df 100644 --- a/ydb/aio/pool.py +++ b/ydb/aio/pool.py @@ -81,10 +81,7 @@ def complete_discovery(self, error: Optional[Exception]) -> None: self._fast_fail_event.set() def remove(self, connection: Connection) -> None: # type: ignore[override] # async Connection type - self.connections_by_node_id.pop(connection.node_id, None) - self.preferred.pop(connection.endpoint, None) - self.connections.pop(connection.endpoint, None) - self.outdated.pop(connection.endpoint, None) + super().remove(connection) # type: ignore[arg-type] # async Connection type if len(self.connections) == 0: self._event.clear() if not self._fast_fail_error: diff --git a/ydb/aio/query/pool_test.py b/ydb/aio/query/pool_test.py index 5317573c..11590e8f 100644 --- a/ydb/aio/query/pool_test.py +++ b/ydb/aio/query/pool_test.py @@ -6,6 +6,7 @@ from ydb import issues from ydb.aio import _utilities as aio_utilities +from ydb.aio.pool import ConnectionsCache from ydb.aio.query.pool import QuerySessionPool from ydb.aio.query.session import QuerySession from ydb.aio.query.transaction import QueryTxContext @@ -401,3 +402,26 @@ async def fake_execute_call(**kwargs): await tx.execute("SELECT 1") self.assertIsNone(captured.get("pool_id")) + + +class TestConnectionCacheCleanup(unittest.IsolatedAsyncioTestCase): + async def test_stale_remove_preserves_replacement(self): + cache = ConnectionsCache() + old = MagicMock(endpoint="localhost:1", node_id=1) + replacement = MagicMock(endpoint=old.endpoint, node_id=old.node_id) + cache.add(old) + cache.remove(old) + cache.add(replacement, preferred=True) + cache.make_outdated(replacement) + + cache.remove(old) + + for mapping, key in ( + (cache.connections, replacement.endpoint), + (cache.connections_by_node_id, replacement.node_id), + (cache.preferred, replacement.endpoint), + (cache.outdated, replacement.endpoint), + ): + self.assertIs(mapping.get(key), replacement) + self.assertTrue(cache._event.is_set()) + self.assertTrue(cache._fast_fail_event.is_set()) diff --git a/ydb/connection.py b/ydb/connection.py index 052b563e..47dbdeed 100644 --- a/ydb/connection.py +++ b/ydb/connection.py @@ -478,7 +478,7 @@ def _finish_call(self, call_state): # Call successfully finished if self.closing and self.calls == 0: # Channel is closing and we have to destroy channel - self.destroy() + threading.Thread(target=self.destroy, daemon=True).start() def future( self, @@ -543,6 +543,7 @@ def __call__( :return: A result of computation """ rpc_state, timeout, metadata = self._prepare_call(stub, rpc_name, request, settings) + finish_call = True try: response = rpc_state( request, @@ -556,11 +557,15 @@ def __call__( # NOTE(vgvoleg): for stream results we should also be able to handle disconnects response = _SafeSyncIterator(response, rpc_state, on_disconnected) - return response if wrap_result is None else wrap_result(rpc_state, response, *wrap_args) + result = response if wrap_result is None else wrap_result(rpc_state, response, *wrap_args) + if isinstance(response, _SafeSyncIterator): + finish_call = not response.add_callback(lambda: self._finish_call(rpc_state)) + return result except grpc.RpcError as rpc_error: raise _rpc_error_handler(rpc_state, rpc_error, on_disconnected) finally: - self._finish_call(rpc_state) + if finish_call: + self._finish_call(rpc_state) @classmethod def ready_factory(cls, endpoint, driver_config, ready_timeout=10, endpoint_options=None): @@ -584,7 +589,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.close() - def close(self): + def close(self, force=False): """ Closes the underlying gRPC channel :return: None @@ -596,21 +601,19 @@ def close(self): for callback in self._cleanup_callbacks: callback(self) - # potentially we should cancel in-flight calls here but currently - # it is not required since gRPC can successfully cancel these calls manually. - - if self.calls == 0: + if force: + threading.Thread(target=self.destroy, daemon=True).start() + elif self.calls == 0: # everything is cancelled/completed and channel can be destroyed self.destroy() def destroy(self): - channel = getattr(self, "_channel", None) + with self.lock: + channel, self._channel = self._channel, None + self._stub_instances.clear() if channel is not None and hasattr(channel, "close"): channel.close() - self._stub_instances.clear() - self._channel = None - def ready_future(self): """ Creates a future that tracks underlying gRPC channel is ready diff --git a/ydb/pool.py b/ydb/pool.py index 2ba1c33a..4bb2d3a7 100644 --- a/ydb/pool.py +++ b/ydb/pool.py @@ -7,6 +7,7 @@ from concurrent import futures import collections import random +import weakref from typing import Any, Callable, ContextManager, List, Optional, Set, Tuple, TYPE_CHECKING from . import connection as connection_impl, issues, resolver, _utilities, tracing @@ -28,6 +29,7 @@ class ConnectionsCache: def __init__(self, use_all_nodes: bool = False, tracer: tracing.Tracer = tracing.Tracer(None)) -> None: self.tracer = tracer self.lock = threading.RLock() + self._all_connections: weakref.WeakSet[Connection] = weakref.WeakSet() self.connections: collections.OrderedDict[str, Connection] = collections.OrderedDict() self.connections_by_node_id: collections.OrderedDict[Optional[int], Connection] = collections.OrderedDict() self.outdated: collections.OrderedDict[str, Connection] = collections.OrderedDict() @@ -44,6 +46,7 @@ def add(self, connection: Optional[Connection], preferred: bool = False) -> bool connection.add_cleanup_callback(self.remove) with self.lock: + self._all_connections.add(connection) if preferred: self.preferred[connection.endpoint] = connection @@ -93,15 +96,15 @@ def make_outdated(self, connection: Connection) -> "ConnectionsCache": def cleanup_outdated(self) -> "ConnectionsCache": with self.lock: outdated_connections = list(self.outdated.values()) - for outdated_connection in outdated_connections: - outdated_connection.close() + for outdated_connection in outdated_connections: + outdated_connection.close() return self def cleanup(self) -> None: with self.lock: - actual_connections = list(self.connections.values()) - for connection in actual_connections: - connection.close() + actual_connections = list(self._all_connections) + for connection in actual_connections: + connection.close(force=True) def complete_discovery(self, error: Optional[Exception]) -> None: with self.lock: @@ -155,10 +158,14 @@ def get(self, preferred_endpoint: Optional[EndpointKey] = None) -> Connection: def remove(self, connection: Connection) -> None: with self.lock: - self.connections_by_node_id.pop(connection.node_id, None) - self.preferred.pop(connection.endpoint, None) - self.connections.pop(connection.endpoint, None) - self.outdated.pop(connection.endpoint, None) + for cache, key in ( + (self.connections_by_node_id, connection.node_id), + (self.preferred, connection.endpoint), + (self.connections, connection.endpoint), + (self.outdated, connection.endpoint), + ): + if cache.get(key) is connection: + cache.pop(key) def get_connection_by_node_id(self, node_id: Optional[int]) -> Optional[Connection]: with self.lock: diff --git a/ydb/query/pool_test.py b/ydb/query/pool_test.py index 6dd1677b..8d2b817d 100644 --- a/ydb/query/pool_test.py +++ b/ydb/query/pool_test.py @@ -5,11 +5,12 @@ import threading import time import unittest -from unittest.mock import MagicMock +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock from unittest.mock import patch -from ydb import _utilities, issues +from ydb import _utilities, connection, issues, pool from ydb.convert import _ResultSet, aggregate_result_sets_by_index, aggregate_result_sets_by_index_async from ydb.query.base import create_execute_query_request from ydb.query.pool import QuerySessionPool @@ -416,3 +417,307 @@ def fake_execute_call(**kwargs): tx.execute("SELECT 1") self.assertIsNone(captured.get("pool_id")) + + +class _StreamCall: + def __init__(self, completed=False, complete_on_subscribe=False): + self.messages = iter((1, 2)) + self.completed = completed + self.complete_on_subscribe = complete_on_subscribe + self.callbacks = [] + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.messages) + except StopIteration: + self.complete() + raise + + def add_callback(self, callback): + if self.completed: + return False + self.callbacks.append(callback) + if self.complete_on_subscribe: + self.complete() + return True + + def complete(self): + self.completed = True + callbacks, self.callbacks = self.callbacks, [] + for callback in callbacks: + callback() + + def cancel(self): + self.complete() + + +class TestStreamConnectionCleanup(unittest.TestCase): + def setUp(self): + self.channel = Mock() + self.closed = threading.Event() + self.channel.close.side_effect = self.closed.set + for name, value in (("channel_factory", lambda *args, **kwargs: self.channel), ("_stubs_list", ())): + patched = patch.object(connection, name, value) + patched.start() + self.addCleanup(patched.stop) + metadata = patch.object(connection, "_construct_metadata", return_value=[]) + metadata.start() + self.addCleanup(metadata.stop) + self.connection = connection.Connection("localhost:1") + self.cache = pool.ConnectionsCache() + self.cache.add(self.connection) + + def call(self, response, wrap_result=None): + stub = type("Stub", (), {}) + self.connection._stub_instances[stub] = SimpleNamespace(ReadTable=lambda *args, **kwargs: response) + return self.connection(SimpleNamespace(), stub, "ReadTable", wrap_result=wrap_result) + + def cleanup_outdated(self): + self.cache.make_outdated(self.connection) + self.cache.cleanup_outdated() + self.assertFalse(self.cache.connections) + self.assertFalse(self.cache.outdated) + + def assert_closed(self): + self.assertTrue(self.closed.wait(5), "channel was not closed") + self.channel.close.assert_called_once_with() + + def assert_finished(self): + self.assertEqual(self.connection.calls, 0) + self.assertFalse(self.connection._call_states) + + def test_cleanup_preserves_active_stream(self): + stream = self.call(_StreamCall()) + self.assertEqual(next(stream), 1) + self.cleanup_outdated() + self.channel.close.assert_not_called() + self.assertEqual(list(stream), [2]) + self.assert_finished() + self.assert_closed() + + def test_cancellation_releases_connection(self): + stream = self.call(_StreamCall()) + self.cleanup_outdated() + self.channel.close.assert_not_called() + stream.cancel() + self.assert_finished() + self.assert_closed() + + def test_rpc_completion_releases_unconsumed_stream(self): + response = _StreamCall() + stream = self.call(response) + self.cleanup_outdated() + self.channel.close.assert_not_called() + response.complete() + self.assert_finished() + self.assert_closed() + self.assertEqual(list(stream), [1, 2]) + + def test_cleanup_waits_for_all_streams(self): + first, second = self.call(_StreamCall()), self.call(_StreamCall()) + self.cleanup_outdated() + first.cancel() + self.assertEqual(self.connection.calls, 1) + self.channel.close.assert_not_called() + second.cancel() + self.assert_finished() + self.assert_closed() + + def test_terminal_cleanup_cancels_active_stream(self): + response = _StreamCall() + self.call(response) + self.assertEqual(self.connection.calls, 1) + + def close_channel(): + response.cancel() + self.closed.set() + + self.channel.close.side_effect = close_channel + self.cache.cleanup() + self.assert_closed() + self.assertTrue(response.completed) + self.assert_finished() + self.assertFalse(self.cache.connections) + + def test_terminal_cleanup_cancels_retired_stream(self): + response = _StreamCall() + self.call(response) + self.cleanup_outdated() + self.channel.close.assert_not_called() + + def close_channel(): + response.cancel() + self.closed.set() + + self.channel.close.side_effect = close_channel + self.cache.cleanup() + self.assert_closed() + self.assertTrue(response.completed) + self.assert_finished() + + def test_stale_remove_preserves_replacement(self): + response = _StreamCall() + self.call(response) + self.cleanup_outdated() + with patch.object(connection, "channel_factory", return_value=Mock()): + replacement = connection.Connection(self.connection.endpoint) + self.cache.add(replacement, preferred=True) + self.cache.make_outdated(replacement) + self.connection.close() + for mapping, key in ( + (self.cache.connections, replacement.endpoint), + (self.cache.connections_by_node_id, replacement.node_id), + (self.cache.preferred, replacement.endpoint), + (self.cache.outdated, replacement.endpoint), + ): + self.assertIs(mapping.get(key), replacement) + response.cancel() + self.assert_closed() + + def test_already_completed_rpc_releases_once(self): + self.call(_StreamCall(completed=True)) + self.assert_finished() + self.cleanup_outdated() + self.assert_closed() + + def test_rpc_completing_during_subscription_releases_once(self): + self.call(_StreamCall(complete_on_subscribe=True)) + self.assert_finished() + self.cleanup_outdated() + self.assert_closed() + + def test_unary_call_releases_connection(self): + self.assertEqual(self.call(42), 42) + self.assert_finished() + self.cleanup_outdated() + self.assert_closed() + + def test_wrapper_failure_releases_connection(self): + def fail(*args): + raise ValueError("invalid response") + + with self.assertRaisesRegex(ValueError, "invalid response"): + self.call(_StreamCall(), wrap_result=fail) + self.assert_finished() + self.cleanup_outdated() + self.assert_closed() + + def test_completion_callback_does_not_wait_for_channel_drain(self): + response = _StreamCall() + self.call(response) + self.cleanup_outdated() + self.channel.close.assert_not_called() + callback_returned = threading.Event() + errors = [] + + def close_channel(): + if not callback_returned.wait(5): + errors.append("channel.close blocked its completion callback") + self.closed.set() + + self.channel.close.side_effect = close_channel + + def complete(): + try: + response.complete() + finally: + callback_returned.set() + + completion = threading.Thread(target=complete, daemon=True) + completion.start() + self.assertTrue(callback_returned.wait(5), "completion callback blocked on channel.close") + completion.join(5) + self.assertFalse(completion.is_alive()) + self.assert_closed() + self.assertFalse(errors, errors) + self.assert_finished() + + def test_terminal_cleanup_callback_does_not_wait_for_channel_drain(self): + response, active = _StreamCall(), _StreamCall() + self.call(response) + self.call(active) + response.add_callback(self.cache.cleanup) + callback_returned = threading.Event() + errors = [] + + def close_channel(): + if not callback_returned.wait(5): + errors.append("channel.close blocked its completion callback") + active.cancel() + self.closed.set() + + self.channel.close.side_effect = close_channel + + def complete(): + try: + response.complete() + finally: + callback_returned.set() + + completion = threading.Thread(target=complete, daemon=True) + completion.start() + self.assertTrue(callback_returned.wait(5), "completion callback blocked on channel.close") + completion.join(5) + self.assertFalse(completion.is_alive()) + self.assert_closed() + self.assertFalse(errors, errors) + self.assert_finished() + + +def test_connection_cache_cleanup_during_disconnect(monkeypatch): + channel = Mock() + channel_closed = threading.Event() + channel.close.side_effect = channel_closed.set + monkeypatch.setattr(connection, "channel_factory", lambda *args, **kwargs: channel) + monkeypatch.setattr(connection, "_stubs_list", []) + cached_connection = connection.Connection("localhost:1") + cache = pool.ConnectionsCache() + disconnect_entered = threading.Event() + stop_entered = threading.Event() + errors = [] + + def before_remove(_): + if threading.current_thread() is disconnect_thread: + disconnect_entered.set() + assert stop_entered.wait(5), "driver cleanup did not reach connection.close" + + cached_connection.add_cleanup_callback(before_remove) + cache.add(cached_connection) + cache.make_outdated(cached_connection) + original_close = connection.Connection.close + + def close(self, *args, **kwargs): + if threading.current_thread() is stop_thread: + stop_entered.set() + return original_close(self, *args, **kwargs) + + monkeypatch.setattr(connection.Connection, "close", close) + + def run(action): + try: + action() + except BaseException as error: + errors.append(error) + + disconnect_thread = threading.Thread(target=run, args=(cached_connection.close,), daemon=True) + stop_thread = threading.Thread( + target=run, + args=(cache.cleanup,), + daemon=True, + ) + disconnect_thread.start() + assert disconnect_entered.wait(5), "disconnect did not reach its cleanup callback" + stop_thread.start() + disconnect_thread.join(5) + stop_thread.join(5) + assert not disconnect_thread.is_alive(), "disconnect deadlocked with driver shutdown" + assert not stop_thread.is_alive(), "driver shutdown deadlocked with disconnect" + assert not errors, errors + assert not cache.connections + assert not cache.connections_by_node_id + assert not cache.outdated + assert channel_closed.wait(5), "driver shutdown did not close the channel" + channel.close.assert_called_once() From 0b74e7b3dfb81538dbe51434a41137223bd35809 Mon Sep 17 00:00:00 2001 From: mregrock <78217843+mregrock@users.noreply.github.com> Date: Thu, 5 Nov 2026 09:02:30 +0000 Subject: [PATCH 2/2] Keep cache identity checks type safe --- ydb/pool.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ydb/pool.py b/ydb/pool.py index 4bb2d3a7..126e89b5 100644 --- a/ydb/pool.py +++ b/ydb/pool.py @@ -158,14 +158,11 @@ def get(self, preferred_endpoint: Optional[EndpointKey] = None) -> Connection: def remove(self, connection: Connection) -> None: with self.lock: - for cache, key in ( - (self.connections_by_node_id, connection.node_id), - (self.preferred, connection.endpoint), - (self.connections, connection.endpoint), - (self.outdated, connection.endpoint), - ): - if cache.get(key) is connection: - cache.pop(key) + if self.connections_by_node_id.get(connection.node_id) is connection: + self.connections_by_node_id.pop(connection.node_id) + for cache in (self.preferred, self.connections, self.outdated): + if cache.get(connection.endpoint) is connection: + cache.pop(connection.endpoint) def get_connection_by_node_id(self, node_id: Optional[int]) -> Optional[Connection]: with self.lock: