Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
* Keep synchronous streams alive during discovery cleanup and safely close their connections on driver shutdown.
* Fix topic reader `close()` hanging forever when the stream is broken with uncommitted offsets pending: the commit flush now stops as soon as the stream reports an error instead of waiting for acks that can never arrive
* Fix async table session pool capacity leaking when keeping an idle session alive fails or is cancelled

Expand Down
5 changes: 1 addition & 4 deletions ydb/aio/pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,7 @@ def complete_discovery(self, error: Optional[Exception]) -> None:
self._fast_fail_event.set()

def remove(self, connection: Connection) -> None: # type: ignore[override] # async Connection type
self.connections_by_node_id.pop(connection.node_id, None)
self.preferred.pop(connection.endpoint, None)
self.connections.pop(connection.endpoint, None)
self.outdated.pop(connection.endpoint, None)
super().remove(connection) # type: ignore[arg-type] # async Connection type
if len(self.connections) == 0:
self._event.clear()
if not self._fast_fail_error:
Expand Down
24 changes: 24 additions & 0 deletions ydb/aio/query/pool_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from ydb import issues
from ydb.aio import _utilities as aio_utilities
from ydb.aio.pool import ConnectionsCache
from ydb.aio.query.pool import QuerySessionPool
from ydb.aio.query.session import QuerySession
from ydb.aio.query.transaction import QueryTxContext
Expand Down Expand Up @@ -401,3 +402,26 @@ async def fake_execute_call(**kwargs):
await tx.execute("SELECT 1")

self.assertIsNone(captured.get("pool_id"))


class TestConnectionCacheCleanup(unittest.IsolatedAsyncioTestCase):
async def test_stale_remove_preserves_replacement(self):
cache = ConnectionsCache()
old = MagicMock(endpoint="localhost:1", node_id=1)
replacement = MagicMock(endpoint=old.endpoint, node_id=old.node_id)
cache.add(old)
cache.remove(old)
cache.add(replacement, preferred=True)
cache.make_outdated(replacement)

cache.remove(old)

for mapping, key in (
(cache.connections, replacement.endpoint),
(cache.connections_by_node_id, replacement.node_id),
(cache.preferred, replacement.endpoint),
(cache.outdated, replacement.endpoint),
):
self.assertIs(mapping.get(key), replacement)
self.assertTrue(cache._event.is_set())
self.assertTrue(cache._fast_fail_event.is_set())
27 changes: 15 additions & 12 deletions ydb/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,7 @@ def _finish_call(self, call_state):
# Call successfully finished
if self.closing and self.calls == 0:
# Channel is closing and we have to destroy channel
self.destroy()
threading.Thread(target=self.destroy, daemon=True).start()

def future(
self,
Expand Down Expand Up @@ -543,6 +543,7 @@ def __call__(
:return: A result of computation
"""
rpc_state, timeout, metadata = self._prepare_call(stub, rpc_name, request, settings)
finish_call = True
try:
response = rpc_state(
request,
Expand All @@ -556,11 +557,15 @@ def __call__(
# NOTE(vgvoleg): for stream results we should also be able to handle disconnects
response = _SafeSyncIterator(response, rpc_state, on_disconnected)

return response if wrap_result is None else wrap_result(rpc_state, response, *wrap_args)
result = response if wrap_result is None else wrap_result(rpc_state, response, *wrap_args)
if isinstance(response, _SafeSyncIterator):
finish_call = not response.add_callback(lambda: self._finish_call(rpc_state))
return result
except grpc.RpcError as rpc_error:
raise _rpc_error_handler(rpc_state, rpc_error, on_disconnected)
finally:
self._finish_call(rpc_state)
if finish_call:
self._finish_call(rpc_state)

@classmethod
def ready_factory(cls, endpoint, driver_config, ready_timeout=10, endpoint_options=None):
Expand All @@ -584,7 +589,7 @@ def __enter__(self):
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()

def close(self):
def close(self, force=False):
"""
Closes the underlying gRPC channel
:return: None
Expand All @@ -596,21 +601,19 @@ def close(self):
for callback in self._cleanup_callbacks:
callback(self)

# potentially we should cancel in-flight calls here but currently
# it is not required since gRPC can successfully cancel these calls manually.

if self.calls == 0:
if force:
threading.Thread(target=self.destroy, daemon=True).start()
elif self.calls == 0:
# everything is cancelled/completed and channel can be destroyed
self.destroy()

def destroy(self):
channel = getattr(self, "_channel", None)
with self.lock:
channel, self._channel = self._channel, None
self._stub_instances.clear()
if channel is not None and hasattr(channel, "close"):
channel.close()

self._stub_instances.clear()
self._channel = None

def ready_future(self):
"""
Creates a future that tracks underlying gRPC channel is ready
Expand Down
22 changes: 13 additions & 9 deletions ydb/pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from concurrent import futures
import collections
import random
import weakref
from typing import Any, Callable, ContextManager, List, Optional, Set, Tuple, TYPE_CHECKING

from . import connection as connection_impl, issues, resolver, _utilities, tracing
Expand All @@ -28,6 +29,7 @@ class ConnectionsCache:
def __init__(self, use_all_nodes: bool = False, tracer: tracing.Tracer = tracing.Tracer(None)) -> None:
self.tracer = tracer
self.lock = threading.RLock()
self._all_connections: weakref.WeakSet[Connection] = weakref.WeakSet()
self.connections: collections.OrderedDict[str, Connection] = collections.OrderedDict()
self.connections_by_node_id: collections.OrderedDict[Optional[int], Connection] = collections.OrderedDict()
self.outdated: collections.OrderedDict[str, Connection] = collections.OrderedDict()
Expand All @@ -44,6 +46,7 @@ def add(self, connection: Optional[Connection], preferred: bool = False) -> bool

connection.add_cleanup_callback(self.remove)
with self.lock:
self._all_connections.add(connection)
if preferred:
self.preferred[connection.endpoint] = connection

Expand Down Expand Up @@ -93,15 +96,15 @@ def make_outdated(self, connection: Connection) -> "ConnectionsCache":
def cleanup_outdated(self) -> "ConnectionsCache":
with self.lock:
outdated_connections = list(self.outdated.values())
for outdated_connection in outdated_connections:
outdated_connection.close()
for outdated_connection in outdated_connections:
outdated_connection.close()
return self

def cleanup(self) -> None:
with self.lock:
actual_connections = list(self.connections.values())
for connection in actual_connections:
connection.close()
actual_connections = list(self._all_connections)
for connection in actual_connections:
connection.close(force=True)

def complete_discovery(self, error: Optional[Exception]) -> None:
with self.lock:
Expand Down Expand Up @@ -155,10 +158,11 @@ def get(self, preferred_endpoint: Optional[EndpointKey] = None) -> Connection:

def remove(self, connection: Connection) -> None:
with self.lock:
self.connections_by_node_id.pop(connection.node_id, None)
self.preferred.pop(connection.endpoint, None)
self.connections.pop(connection.endpoint, None)
self.outdated.pop(connection.endpoint, None)
if self.connections_by_node_id.get(connection.node_id) is connection:
self.connections_by_node_id.pop(connection.node_id)
for cache in (self.preferred, self.connections, self.outdated):
if cache.get(connection.endpoint) is connection:
cache.pop(connection.endpoint)

def get_connection_by_node_id(self, node_id: Optional[int]) -> Optional[Connection]:
with self.lock:
Expand Down
Loading
Loading