From d07887a64839f7ce5171b6cbb80cd2120aac194d Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Fri, 23 Oct 2026 14:21:08 +0300 Subject: [PATCH 1/2] Fix stream recovery when gRPC channels close Treat transport-originated stream cancellation as ConnectionLost so existing retry and reconnect handling can recover. Preserve explicit caller and stream cancellation, and cover Query and Topic streams with real gRPC regressions. --- CHANGELOG.md | 2 + ydb/_utilities_test.py | 191 +++++++++++++++++++++++++++++++++++++++++ ydb/aio/connection.py | 23 ++++- ydb/connection.py | 11 ++- 4 files changed, 220 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a1c411..e2482d06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Fix Query and async Topic recovery when discovery closes a gRPC channel with an active stream, preserving explicit caller cancellation + ## 3.33.2 ## * Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried diff --git a/ydb/_utilities_test.py b/ydb/_utilities_test.py index a154afd4..d28bf6c8 100644 --- a/ydb/_utilities_test.py +++ b/ydb/_utilities_test.py @@ -1,12 +1,19 @@ from pathlib import Path, PurePosixPath +import asyncio +from concurrent.futures import ThreadPoolExecutor import subprocess import sys import tarfile +import threading import zipfile +import grpc import pytest import ydb +from ydb.aio.connection import _SafeAsyncIterator +from ydb.aio.connection import Connection as AsyncConnection +from ydb.connection import Connection as SyncConnection from ydb._utilities import check_module_exists from ydb._utilities import get_first_message_with_timeout @@ -50,6 +57,190 @@ def stream(): assert exc_info.value is error +@pytest.fixture +async def grpc_stream_endpoint(): + def responses(request, context): + finished = threading.Event() + context.add_callback(finished.set) + yield request + if request == b"cancel": + context.abort(grpc.StatusCode.CANCELLED, "server cancelled") + assert finished.wait(timeout=5) + + def bidirectional(requests, context): + yield from responses(next(requests), context) + + with ThreadPoolExecutor(max_workers=4) as executor: + server = grpc.server(executor) + server.add_generic_rpc_handlers( + ( + grpc.method_handlers_generic_handler( + "test.Transport", + { + "Query": grpc.unary_stream_rpc_method_handler(responses), + "Topic": grpc.stream_stream_rpc_method_handler(bidirectional), + }, + ), + ) + ) + port = server.add_insecure_port("127.0.0.1:0") + server.start() + try: + yield f"127.0.0.1:{port}" + finally: + assert await asyncio.to_thread(server.stop(0).wait, timeout=5) + + +class _StreamTestStub: + def __init__(self, channel): + self.Query = channel.unary_stream("/test.Transport/Query") + self.Topic = channel.stream_stream("/test.Transport/Topic") + + +@pytest.mark.parametrize("notify", [False, True]) +@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server"]) +def test_sync_stream_distinguishes_channel_close_from_cancellation(grpc_stream_endpoint, cancellation, notify): + disconnected = [] + with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + stream = connection( + b"cancel" if cancellation == "server" else b"first", + _StreamTestStub, + "Query", + on_disconnected=(lambda: disconnected.append(True)) if notify else None, + ) + assert next(stream) in (b"first", b"cancel") + + if cancellation in ("call", "call-and-channel"): + stream.cancel() + if cancellation in ("channel", "call-and-channel"): + connection.close() + + error = ydb.ConnectionLost if cancellation == "channel" else ydb.Cancelled + with pytest.raises(error): + next(stream) + assert disconnected == ([True] if cancellation == "channel" and notify else []) + + +def test_sync_retry_recovers_after_channel_closes_during_stream(grpc_stream_endpoint): + attempts = [] + + def operation(): + with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + stream = connection(b"first", _StreamTestStub, "Query", on_disconnected=lambda: None) + attempts.append(next(stream)) + if len(attempts) == 1: + connection.close() + next(stream) + stream.cancel() + return attempts[-1] + + result = ydb.retry_operation_sync( + operation, + retry_settings=ydb.RetrySettings( + idempotent=True, max_retries=1, slow_backoff_settings=ydb.BackoffSettings(slot_duration=0) + ), + ) + assert result == b"first" + assert attempts == [b"first", b"first"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["Query", "Topic"]) +@pytest.mark.parametrize("notify", [False, True]) +@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server"]) +async def test_async_stream_distinguishes_channel_close_from_cancellation( + grpc_stream_endpoint, method, cancellation, notify +): + disconnected = [] + + async def on_disconnected(): + disconnected.append(True) + + async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + request = b"cancel" if cancellation == "server" else b"first" + stream = await connection( + request if method == "Query" else iter([request]), + _StreamTestStub, + method, + on_disconnected=on_disconnected if notify else None, + ) + assert await asyncio.wait_for(stream.__anext__(), timeout=5) == request + + if cancellation in ("call", "call-and-channel"): + stream.cancel() + if cancellation in ("channel", "call-and-channel"): + await connection.close() + + error = { + "channel": ydb.ConnectionLost, + "call": asyncio.CancelledError, + "call-and-channel": asyncio.CancelledError, + "server": ydb.Cancelled, + }[cancellation] + with pytest.raises(error): + await asyncio.wait_for(stream.__anext__(), timeout=5) + assert disconnected == ([True] if cancellation == "channel" and notify else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("close_channel", [False, True]) +async def test_async_stream_preserves_caller_task_cancellation(grpc_stream_endpoint, close_channel): + entered = asyncio.Event() + disconnected = [] + + async def on_disconnected(): + disconnected.append(True) + + async with grpc.aio.insecure_channel(grpc_stream_endpoint) as channel: + response = channel.unary_stream("/test.Transport/Query")(b"first") + assert await asyncio.wait_for(response.__aiter__().__anext__(), timeout=5) == b"first" + + async def observed(): + entered.set() + async for message in response: + yield message + + stream = _SafeAsyncIterator(observed(), "test", on_disconnected) + task = asyncio.create_task(stream.__anext__()) + try: + await asyncio.wait_for(entered.wait(), timeout=5) + task.cancel() + if close_channel: + await channel.close() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=5) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert disconnected == [] + + +@pytest.mark.asyncio +async def test_async_retry_recovers_after_channel_closes_during_stream(grpc_stream_endpoint): + from ydb.retries import retry_operation_async + + attempts = [] + + async def operation(): + async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + stream = await connection(b"first", _StreamTestStub, "Query") + attempts.append(await asyncio.wait_for(stream.__anext__(), timeout=5)) + if len(attempts) == 1: + await connection.close() + await asyncio.wait_for(stream.__anext__(), timeout=5) + stream.cancel() + return attempts[-1] + + result = await retry_operation_async( + operation, + retry_settings=ydb.RetrySettings( + idempotent=True, max_retries=1, slow_backoff_settings=ydb.BackoffSettings(slot_duration=0) + ), + ) + assert result == b"first" + assert attempts == [b"first", b"first"] + + def test_convert_floats_to_embedding_bytes_encodes_float32_values(): assert ydb.convert_floats_to_embedding_bytes([1, -2, 0.5]) == bytes.fromhex("0000803f000000c00000003f01") diff --git a/ydb/aio/connection.py b/ydb/aio/connection.py index 3bc72f92..18dee510 100644 --- a/ydb/aio/connection.py +++ b/ydb/aio/connection.py @@ -120,8 +120,10 @@ def __init__(self, resp, rpc_state, on_disconnected_callback): self.it = resp.__aiter__() self.rpc_state = rpc_state self.on_disconnected_callback = on_disconnected_callback + self._cancelled = False def cancel(self): + self._cancelled = True self.resp.cancel() return self @@ -129,13 +131,26 @@ def __aiter__(self): return self async def __anext__(self): + next_response = asyncio.ensure_future(self.it.__anext__()) try: - return await self.it.__anext__() + # wait() keeps caller cancellation separate from gRPC cancelling its + # own read when discovery retires the underlying channel. + await asyncio.wait([next_response]) + except asyncio.CancelledError: + next_response.cancel() + await asyncio.gather(next_response, return_exceptions=True) + raise + try: + return next_response.result() + except asyncio.CancelledError: + if self._cancelled: + raise + ydb_error = issues.ConnectionLost("gRPC stream cancelled") except grpc.RpcError as rpc_error: ydb_error = _rpc_error_handler(self.rpc_state, rpc_error, use_unavailable=True) - if _is_disconnect_needed(ydb_error): - await self.on_disconnected_callback() - raise ydb_error + if _is_disconnect_needed(ydb_error) and self.on_disconnected_callback is not None: + await self.on_disconnected_callback() + raise ydb_error def __getattr__(self, item): return getattr(self.resp, item) diff --git a/ydb/connection.py b/ydb/connection.py index 052b563e..2b0fd32f 100644 --- a/ydb/connection.py +++ b/ydb/connection.py @@ -374,13 +374,16 @@ def __init__(self, endpoint, node_id): class _SafeSyncIterator: - def __init__(self, resp, rpc_state, on_disconnected_callback): + def __init__(self, resp, rpc_state, on_disconnected_callback, connection): self.resp = resp self.it = resp.__iter__() self.rpc_state = rpc_state self.on_disconnected_callback = on_disconnected_callback + self.connection = connection + self._cancelled = False def cancel(self): + self._cancelled = True self.resp.cancel() return self @@ -392,7 +395,9 @@ def __next__(self): return self.it.__next__() except grpc.RpcError as rpc_error: ydb_error = _rpc_error_handler(self.rpc_state, rpc_error, use_unavailable=True) - if _is_disconnect_needed(ydb_error): + if self.connection.closing and not self._cancelled and isinstance(ydb_error, issues.Cancelled): + ydb_error = issues.ConnectionLost(rpc_error.details()) + if _is_disconnect_needed(ydb_error) and self.on_disconnected_callback is not None: self.on_disconnected_callback() raise ydb_error @@ -554,7 +559,7 @@ def __call__( if hasattr(response, "__iter__"): # NOTE(vgvoleg): for stream results we should also be able to handle disconnects - response = _SafeSyncIterator(response, rpc_state, on_disconnected) + response = _SafeSyncIterator(response, rpc_state, on_disconnected, self) return response if wrap_result is None else wrap_result(rpc_state, response, *wrap_args) except grpc.RpcError as rpc_error: From 31e0632ff28356bc5bb949de7edda7ec613bc665 Mon Sep 17 00:00:00 2001 From: KirillKurdyukov Date: Tue, 27 Oct 2026 11:52:47 +0300 Subject: [PATCH 2/2] Distinguish transport cancellation and retry idempotent operations --- CHANGELOG.md | 2 +- docs/apireference.rst | 3 + docs/errors.rst | 19 +++++- ydb/_errors.py | 5 ++ ydb/_utilities_test.py | 143 ++++++++++++++++++++++++++++++++--------- ydb/aio/connection.py | 2 +- ydb/connection.py | 12 ++-- ydb/issues.py | 5 ++ 8 files changed, 149 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2482d06..86e3b744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -* Fix Query and async Topic recovery when discovery closes a gRPC channel with an active stream, preserving explicit caller cancellation +* Distinguish gRPC cancellation as `ClientCancelled` and retry it only for idempotent operations, preserving explicit caller cancellation and non-retriable deadlines ## 3.33.2 ## * Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried diff --git a/docs/apireference.rst b/docs/apireference.rst index 5dc3144c..67ccc550 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -613,6 +613,9 @@ Errors .. autoclass:: ydb.Undetermined :undoc-members: +.. autoclass:: ydb.ClientCancelled + :undoc-members: + .. autoclass:: ydb.ConnectionError :undoc-members: diff --git a/docs/errors.rst b/docs/errors.rst index 77a9e88c..eb972436 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -27,7 +27,8 @@ Exception Hierarchy ├── ydb.AlreadyExists — object already exists (DDL) ├── ydb.NotFound — object not found ├── ydb.SessionExpired — session TTL expired - ├── ydb.Cancelled — operation was cancelled + ├── ydb.Cancelled — server status or explicit sync stream cancellation + │ └── ydb.ClientCancelled — gRPC call was cancelled ├── ydb.Undetermined — outcome is unknown (e.g. network lost before ack) ├── ydb.Unsupported — operation not supported by this server version ├── ydb.SessionBusy — session is executing another request @@ -41,6 +42,13 @@ Exception Hierarchy │ └── ydb.Unimplemented — server does not support this RPC └── ydb.DeadlineExceed — client-side deadline exceeded +``ClientCancelled`` has the distinct status ``StatusCode.CLIENT_CANCELLED``. It +inherits from ``Cancelled`` so existing cancellation handlers continue to catch +it, but its retries are controlled by ``idempotent``, not ``retry_cancelled``. +The server's YDB ``CANCELLED`` status remains ``Cancelled`` with +``StatusCode.CANCELLED``. Explicit async stream or caller-task cancellation +continues to raise ``asyncio.CancelledError``. + Catching Errors --------------- @@ -100,6 +108,12 @@ Not every error is worth retrying. The SDK classifies errors into three groups: * :class:`~ydb.SessionPoolEmpty` — all pool sessions busy; wait and retry. * :class:`~ydb.ConnectionError` / :class:`~ydb.ConnectionLost` — network issues; reconnect and retry. +**Retriable only for idempotent operations (fast backoff):** + +* :class:`~ydb.ClientCancelled` — gRPC ``CANCELLED``, including cancellation caused + by a closing channel. Set ``idempotent=True`` in :class:`~ydb.RetrySettings` + to enable retries. ``retry_cancelled=True`` does not override this requirement. + **Retriable only for idempotent operations (slow backoff):** * :class:`~ydb.Undetermined` — outcome unknown; only safe to retry if the operation is idempotent @@ -108,6 +122,7 @@ Not every error is worth retrying. The SDK classifies errors into three groups: **Never retried:** +* :class:`~ydb.DeadlineExceed` — gRPC ``DEADLINE_EXCEEDED``. * :class:`~ydb.BadRequest`, :class:`~ydb.Unauthorized`, ``Unauthenticated``, :class:`~ydb.SchemeError`, :class:`~ydb.AlreadyExists`, ``Unsupported``, :class:`~ydb.Timeout`, ``PreconditionFailed``, ``ExternalError`` — these indicate @@ -126,7 +141,7 @@ All pool methods that perform retries (``execute_with_retries``, ``retry_operati retry = ydb.RetrySettings( max_retries=5, # max retry attempts (default: 10) - idempotent=False, # set True to also retry Undetermined errors + idempotent=False, # set True to retry ClientCancelled and Undetermined retry_cancelled=False, # set True to retry Cancelled errors ) diff --git a/ydb/_errors.py b/ydb/_errors.py index 1e969c09..a0ef0e9d 100644 --- a/ydb/_errors.py +++ b/ydb/_errors.py @@ -22,6 +22,11 @@ def check_retriable_error(err, retry_settings, attempt): + if isinstance(err, issues.ClientCancelled): + if retry_settings.idempotent: + return ErrorRetryInfo(True, retry_settings.fast_backoff.calc_timeout(attempt)) + return ErrorRetryInfo(False, None) + if isinstance(err, issues.Cancelled): if retry_settings.retry_cancelled: return ErrorRetryInfo(True, retry_settings.fast_backoff.calc_timeout(attempt)) diff --git a/ydb/_utilities_test.py b/ydb/_utilities_test.py index d28bf6c8..cb12bf32 100644 --- a/ydb/_utilities_test.py +++ b/ydb/_utilities_test.py @@ -5,6 +5,8 @@ import sys import tarfile import threading +from types import SimpleNamespace +from unittest.mock import Mock import zipfile import grpc @@ -14,7 +16,9 @@ from ydb.aio.connection import _SafeAsyncIterator from ydb.aio.connection import Connection as AsyncConnection from ydb.connection import Connection as SyncConnection +from ydb.connection import _rpc_error_handler +from ydb._errors import check_retriable_error from ydb._utilities import check_module_exists from ydb._utilities import get_first_message_with_timeout from ydb._utilities import x_ydb_sdk_build_info_header @@ -57,6 +61,51 @@ def stream(): assert exc_info.value is error +@pytest.mark.parametrize( + "code,error,status", + [ + (grpc.StatusCode.CANCELLED, ydb.ClientCancelled, ydb.StatusCode.CLIENT_CANCELLED), + (grpc.StatusCode.DEADLINE_EXCEEDED, ydb.DeadlineExceed, ydb.StatusCode.DEADLINE_EXCEEDED), + ], +) +def test_grpc_cancellation_and_deadline_have_distinct_statuses(code, error, status): + rpc_error = Mock(spec=grpc.Call) + rpc_error.code.return_value = code + rpc_error.details.return_value = "transport error" + + result = _rpc_error_handler("test", rpc_error) + + assert type(result) is error + assert result.status == status + if error is ydb.ClientCancelled: + assert isinstance(result, ydb.Cancelled) + + +def test_server_cancellation_preserves_server_status(): + response = SimpleNamespace(status=ydb.StatusCode.CANCELLED, issues=[]) + with pytest.raises(ydb.Cancelled) as exc_info: + ydb.issues._process_response(response) + assert type(exc_info.value) is ydb.Cancelled + assert exc_info.value.status == ydb.StatusCode.CANCELLED + + +@pytest.mark.parametrize("idempotent", [False, True]) +@pytest.mark.parametrize("retry_cancelled", [False, True]) +@pytest.mark.parametrize("error", [ydb.ClientCancelled, ydb.Cancelled, ydb.DeadlineExceed, asyncio.CancelledError]) +def test_cancellation_and_deadline_retry_policy(error, idempotent, retry_cancelled): + retry_settings = ydb.RetrySettings( + idempotent=idempotent, + retry_cancelled=retry_cancelled, + fast_backoff_settings=ydb.BackoffSettings(slot_duration=0), + ) + + result = check_retriable_error(error("cancelled"), retry_settings, 0) + + expected = (error is ydb.ClientCancelled and idempotent) or (error is ydb.Cancelled and retry_cancelled) + assert result.is_retriable is expected + assert result.sleep_timeout_seconds == (0 if expected else None) + + @pytest.fixture async def grpc_stream_endpoint(): def responses(request, context): @@ -65,6 +114,8 @@ def responses(request, context): yield request if request == b"cancel": context.abort(grpc.StatusCode.CANCELLED, "server cancelled") + if request == b"deadline": + context.abort(grpc.StatusCode.DEADLINE_EXCEEDED, "deadline exceeded") assert finished.wait(timeout=5) def bidirectional(requests, context): @@ -98,56 +149,71 @@ def __init__(self, channel): @pytest.mark.parametrize("notify", [False, True]) -@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server"]) +@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server", "deadline"]) def test_sync_stream_distinguishes_channel_close_from_cancellation(grpc_stream_endpoint, cancellation, notify): disconnected = [] with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + request = {"server": b"cancel", "deadline": b"deadline"}.get(cancellation, b"first") stream = connection( - b"cancel" if cancellation == "server" else b"first", + request, _StreamTestStub, "Query", on_disconnected=(lambda: disconnected.append(True)) if notify else None, ) - assert next(stream) in (b"first", b"cancel") + assert next(stream) == request if cancellation in ("call", "call-and-channel"): stream.cancel() if cancellation in ("channel", "call-and-channel"): connection.close() - error = ydb.ConnectionLost if cancellation == "channel" else ydb.Cancelled - with pytest.raises(error): + error = { + "channel": ydb.ClientCancelled, + "call": ydb.Cancelled, + "call-and-channel": ydb.Cancelled, + "server": ydb.ClientCancelled, + "deadline": ydb.DeadlineExceed, + }[cancellation] + with pytest.raises(error) as exc_info: next(stream) - assert disconnected == ([True] if cancellation == "channel" and notify else []) + assert type(exc_info.value) is error + assert disconnected == ([True] if cancellation in ("channel", "server") and notify else []) -def test_sync_retry_recovers_after_channel_closes_during_stream(grpc_stream_endpoint): +@pytest.mark.parametrize("idempotent", [False, True]) +@pytest.mark.parametrize("failure", ["channel", "server", "deadline"]) +def test_sync_retry_handles_transport_cancellation_and_deadline(grpc_stream_endpoint, failure, idempotent): attempts = [] + first_request = {"server": b"cancel", "deadline": b"deadline"}.get(failure, b"first") def operation(): with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: - stream = connection(b"first", _StreamTestStub, "Query", on_disconnected=lambda: None) + stream = connection(first_request if not attempts else b"first", _StreamTestStub, "Query") attempts.append(next(stream)) if len(attempts) == 1: - connection.close() + if failure == "channel": + connection.close() next(stream) stream.cancel() return attempts[-1] - result = ydb.retry_operation_sync( - operation, - retry_settings=ydb.RetrySettings( - idempotent=True, max_retries=1, slow_backoff_settings=ydb.BackoffSettings(slot_duration=0) - ), + retry_settings = ydb.RetrySettings( + idempotent=idempotent, max_retries=1, fast_backoff_settings=ydb.BackoffSettings(slot_duration=0) ) - assert result == b"first" - assert attempts == [b"first", b"first"] + if idempotent and failure != "deadline": + assert ydb.retry_operation_sync(operation, retry_settings=retry_settings) == b"first" + assert attempts == [first_request, b"first"] + else: + error = ydb.DeadlineExceed if failure == "deadline" else ydb.ClientCancelled + with pytest.raises(error): + ydb.retry_operation_sync(operation, retry_settings=retry_settings) + assert attempts == [first_request] @pytest.mark.asyncio @pytest.mark.parametrize("method", ["Query", "Topic"]) @pytest.mark.parametrize("notify", [False, True]) -@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server"]) +@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server", "deadline"]) async def test_async_stream_distinguishes_channel_close_from_cancellation( grpc_stream_endpoint, method, cancellation, notify ): @@ -157,7 +223,7 @@ async def on_disconnected(): disconnected.append(True) async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: - request = b"cancel" if cancellation == "server" else b"first" + request = {"server": b"cancel", "deadline": b"deadline"}.get(cancellation, b"first") stream = await connection( request if method == "Query" else iter([request]), _StreamTestStub, @@ -172,14 +238,16 @@ async def on_disconnected(): await connection.close() error = { - "channel": ydb.ConnectionLost, + "channel": ydb.ClientCancelled, "call": asyncio.CancelledError, "call-and-channel": asyncio.CancelledError, - "server": ydb.Cancelled, + "server": ydb.ClientCancelled, + "deadline": ydb.DeadlineExceed, }[cancellation] - with pytest.raises(error): + with pytest.raises(error) as exc_info: await asyncio.wait_for(stream.__anext__(), timeout=5) - assert disconnected == ([True] if cancellation == "channel" and notify else []) + assert type(exc_info.value) is error + assert disconnected == ([True] if cancellation in ("channel", "server") and notify else []) @pytest.mark.asyncio @@ -216,29 +284,40 @@ async def observed(): @pytest.mark.asyncio -async def test_async_retry_recovers_after_channel_closes_during_stream(grpc_stream_endpoint): +@pytest.mark.parametrize("method", ["Query", "Topic"]) +@pytest.mark.parametrize("idempotent", [False, True]) +@pytest.mark.parametrize("failure", ["channel", "server", "deadline"]) +async def test_async_retry_handles_transport_cancellation_and_deadline( + grpc_stream_endpoint, method, failure, idempotent +): from ydb.retries import retry_operation_async attempts = [] + first_request = {"server": b"cancel", "deadline": b"deadline"}.get(failure, b"first") async def operation(): async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: - stream = await connection(b"first", _StreamTestStub, "Query") + request = first_request if not attempts else b"first" + stream = await connection(request if method == "Query" else iter([request]), _StreamTestStub, method) attempts.append(await asyncio.wait_for(stream.__anext__(), timeout=5)) if len(attempts) == 1: - await connection.close() + if failure == "channel": + await connection.close() await asyncio.wait_for(stream.__anext__(), timeout=5) stream.cancel() return attempts[-1] - result = await retry_operation_async( - operation, - retry_settings=ydb.RetrySettings( - idempotent=True, max_retries=1, slow_backoff_settings=ydb.BackoffSettings(slot_duration=0) - ), + retry_settings = ydb.RetrySettings( + idempotent=idempotent, max_retries=1, fast_backoff_settings=ydb.BackoffSettings(slot_duration=0) ) - assert result == b"first" - assert attempts == [b"first", b"first"] + if idempotent and failure != "deadline": + assert await retry_operation_async(operation, retry_settings=retry_settings) == b"first" + assert attempts == [first_request, b"first"] + else: + error = ydb.DeadlineExceed if failure == "deadline" else ydb.ClientCancelled + with pytest.raises(error): + await retry_operation_async(operation, retry_settings=retry_settings) + assert attempts == [first_request] def test_convert_floats_to_embedding_bytes_encodes_float32_values(): diff --git a/ydb/aio/connection.py b/ydb/aio/connection.py index 18dee510..d05251ab 100644 --- a/ydb/aio/connection.py +++ b/ydb/aio/connection.py @@ -145,7 +145,7 @@ async def __anext__(self): except asyncio.CancelledError: if self._cancelled: raise - ydb_error = issues.ConnectionLost("gRPC stream cancelled") + ydb_error = issues.ClientCancelled("gRPC stream cancelled") except grpc.RpcError as rpc_error: ydb_error = _rpc_error_handler(self.rpc_state, rpc_error, use_unavailable=True) if _is_disconnect_needed(ydb_error) and self.on_disconnected_callback is not None: diff --git a/ydb/connection.py b/ydb/connection.py index 2b0fd32f..00c63c7a 100644 --- a/ydb/connection.py +++ b/ydb/connection.py @@ -99,7 +99,7 @@ def _rpc_error_handler( elif rpc_error.code() == grpc.StatusCode.UNIMPLEMENTED: return issues.Unimplemented("Method or feature is not implemented on server!") elif rpc_error.code() == grpc.StatusCode.CANCELLED: - return issues.Cancelled(rpc_error.details()) + return issues.ClientCancelled(rpc_error.details()) elif rpc_error.code() == grpc.StatusCode.RESOURCE_EXHAUSTED: if "Sent message larger than max" in rpc_error.details(): return issues.BadRequest(rpc_error.details()) @@ -119,6 +119,7 @@ def _is_disconnect_needed(error): ( issues.ConnectionLost, issues.Unavailable, + issues.ClientCancelled, ), ) @@ -374,12 +375,11 @@ def __init__(self, endpoint, node_id): class _SafeSyncIterator: - def __init__(self, resp, rpc_state, on_disconnected_callback, connection): + def __init__(self, resp, rpc_state, on_disconnected_callback): self.resp = resp self.it = resp.__iter__() self.rpc_state = rpc_state self.on_disconnected_callback = on_disconnected_callback - self.connection = connection self._cancelled = False def cancel(self): @@ -395,8 +395,8 @@ def __next__(self): return self.it.__next__() except grpc.RpcError as rpc_error: ydb_error = _rpc_error_handler(self.rpc_state, rpc_error, use_unavailable=True) - if self.connection.closing and not self._cancelled and isinstance(ydb_error, issues.Cancelled): - ydb_error = issues.ConnectionLost(rpc_error.details()) + if self._cancelled and isinstance(ydb_error, issues.ClientCancelled): + ydb_error = issues.Cancelled(rpc_error.details()) if _is_disconnect_needed(ydb_error) and self.on_disconnected_callback is not None: self.on_disconnected_callback() raise ydb_error @@ -559,7 +559,7 @@ def __call__( if hasattr(response, "__iter__"): # NOTE(vgvoleg): for stream results we should also be able to handle disconnects - response = _SafeSyncIterator(response, rpc_state, on_disconnected, self) + response = _SafeSyncIterator(response, rpc_state, on_disconnected) return response if wrap_result is None else wrap_result(rpc_state, response, *wrap_args) except grpc.RpcError as rpc_error: diff --git a/ydb/issues.py b/ydb/issues.py index fba0ee0c..f73c3ebb 100644 --- a/ydb/issues.py +++ b/ydb/issues.py @@ -60,6 +60,7 @@ class StatusCode(enum.IntEnum): DEADLINE_EXCEEDED = _TRANSPORT_STATUSES_FIRST + 30 CLIENT_INTERNAL_ERROR = _TRANSPORT_STATUSES_FIRST + 40 UNIMPLEMENTED = _TRANSPORT_STATUSES_FIRST + 50 + CLIENT_CANCELLED = _TRANSPORT_STATUSES_FIRST + 60 UNAUTHENTICATED = _CLIENT_STATUSES_FIRST + 30 SESSION_POOL_EMPTY = _CLIENT_STATUSES_FIRST + 40 @@ -172,6 +173,10 @@ class Cancelled(Error): status: ClassVar[Optional[StatusCode]] = StatusCode.CANCELLED +class ClientCancelled(Cancelled): + status: ClassVar[Optional[StatusCode]] = StatusCode.CLIENT_CANCELLED + + class Undetermined(Error): status: ClassVar[Optional[StatusCode]] = StatusCode.UNDETERMINED