diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a1c411..86e3b744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Distinguish gRPC cancellation as `ClientCancelled` and retry it only for idempotent operations, preserving explicit caller cancellation and non-retriable deadlines + ## 3.33.2 ## * Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried diff --git a/docs/apireference.rst b/docs/apireference.rst index 5dc3144c..67ccc550 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -613,6 +613,9 @@ Errors .. autoclass:: ydb.Undetermined :undoc-members: +.. autoclass:: ydb.ClientCancelled + :undoc-members: + .. autoclass:: ydb.ConnectionError :undoc-members: diff --git a/docs/errors.rst b/docs/errors.rst index 77a9e88c..eb972436 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -27,7 +27,8 @@ Exception Hierarchy ├── ydb.AlreadyExists — object already exists (DDL) ├── ydb.NotFound — object not found ├── ydb.SessionExpired — session TTL expired - ├── ydb.Cancelled — operation was cancelled + ├── ydb.Cancelled — server status or explicit sync stream cancellation + │ └── ydb.ClientCancelled — gRPC call was cancelled ├── ydb.Undetermined — outcome is unknown (e.g. network lost before ack) ├── ydb.Unsupported — operation not supported by this server version ├── ydb.SessionBusy — session is executing another request @@ -41,6 +42,13 @@ Exception Hierarchy │ └── ydb.Unimplemented — server does not support this RPC └── ydb.DeadlineExceed — client-side deadline exceeded +``ClientCancelled`` has the distinct status ``StatusCode.CLIENT_CANCELLED``. It +inherits from ``Cancelled`` so existing cancellation handlers continue to catch +it, but its retries are controlled by ``idempotent``, not ``retry_cancelled``. +The server's YDB ``CANCELLED`` status remains ``Cancelled`` with +``StatusCode.CANCELLED``. Explicit async stream or caller-task cancellation +continues to raise ``asyncio.CancelledError``. + Catching Errors --------------- @@ -100,6 +108,12 @@ Not every error is worth retrying. The SDK classifies errors into three groups: * :class:`~ydb.SessionPoolEmpty` — all pool sessions busy; wait and retry. * :class:`~ydb.ConnectionError` / :class:`~ydb.ConnectionLost` — network issues; reconnect and retry. +**Retriable only for idempotent operations (fast backoff):** + +* :class:`~ydb.ClientCancelled` — gRPC ``CANCELLED``, including cancellation caused + by a closing channel. Set ``idempotent=True`` in :class:`~ydb.RetrySettings` + to enable retries. ``retry_cancelled=True`` does not override this requirement. + **Retriable only for idempotent operations (slow backoff):** * :class:`~ydb.Undetermined` — outcome unknown; only safe to retry if the operation is idempotent @@ -108,6 +122,7 @@ Not every error is worth retrying. The SDK classifies errors into three groups: **Never retried:** +* :class:`~ydb.DeadlineExceed` — gRPC ``DEADLINE_EXCEEDED``. * :class:`~ydb.BadRequest`, :class:`~ydb.Unauthorized`, ``Unauthenticated``, :class:`~ydb.SchemeError`, :class:`~ydb.AlreadyExists`, ``Unsupported``, :class:`~ydb.Timeout`, ``PreconditionFailed``, ``ExternalError`` — these indicate @@ -126,7 +141,7 @@ All pool methods that perform retries (``execute_with_retries``, ``retry_operati retry = ydb.RetrySettings( max_retries=5, # max retry attempts (default: 10) - idempotent=False, # set True to also retry Undetermined errors + idempotent=False, # set True to retry ClientCancelled and Undetermined retry_cancelled=False, # set True to retry Cancelled errors ) diff --git a/ydb/_errors.py b/ydb/_errors.py index 1e969c09..a0ef0e9d 100644 --- a/ydb/_errors.py +++ b/ydb/_errors.py @@ -22,6 +22,11 @@ def check_retriable_error(err, retry_settings, attempt): + if isinstance(err, issues.ClientCancelled): + if retry_settings.idempotent: + return ErrorRetryInfo(True, retry_settings.fast_backoff.calc_timeout(attempt)) + return ErrorRetryInfo(False, None) + if isinstance(err, issues.Cancelled): if retry_settings.retry_cancelled: return ErrorRetryInfo(True, retry_settings.fast_backoff.calc_timeout(attempt)) diff --git a/ydb/_utilities_test.py b/ydb/_utilities_test.py index a154afd4..cb12bf32 100644 --- a/ydb/_utilities_test.py +++ b/ydb/_utilities_test.py @@ -1,13 +1,24 @@ from pathlib import Path, PurePosixPath +import asyncio +from concurrent.futures import ThreadPoolExecutor import subprocess import sys import tarfile +import threading +from types import SimpleNamespace +from unittest.mock import Mock import zipfile +import grpc import pytest import ydb +from ydb.aio.connection import _SafeAsyncIterator +from ydb.aio.connection import Connection as AsyncConnection +from ydb.connection import Connection as SyncConnection +from ydb.connection import _rpc_error_handler +from ydb._errors import check_retriable_error from ydb._utilities import check_module_exists from ydb._utilities import get_first_message_with_timeout from ydb._utilities import x_ydb_sdk_build_info_header @@ -50,6 +61,265 @@ def stream(): assert exc_info.value is error +@pytest.mark.parametrize( + "code,error,status", + [ + (grpc.StatusCode.CANCELLED, ydb.ClientCancelled, ydb.StatusCode.CLIENT_CANCELLED), + (grpc.StatusCode.DEADLINE_EXCEEDED, ydb.DeadlineExceed, ydb.StatusCode.DEADLINE_EXCEEDED), + ], +) +def test_grpc_cancellation_and_deadline_have_distinct_statuses(code, error, status): + rpc_error = Mock(spec=grpc.Call) + rpc_error.code.return_value = code + rpc_error.details.return_value = "transport error" + + result = _rpc_error_handler("test", rpc_error) + + assert type(result) is error + assert result.status == status + if error is ydb.ClientCancelled: + assert isinstance(result, ydb.Cancelled) + + +def test_server_cancellation_preserves_server_status(): + response = SimpleNamespace(status=ydb.StatusCode.CANCELLED, issues=[]) + with pytest.raises(ydb.Cancelled) as exc_info: + ydb.issues._process_response(response) + assert type(exc_info.value) is ydb.Cancelled + assert exc_info.value.status == ydb.StatusCode.CANCELLED + + +@pytest.mark.parametrize("idempotent", [False, True]) +@pytest.mark.parametrize("retry_cancelled", [False, True]) +@pytest.mark.parametrize("error", [ydb.ClientCancelled, ydb.Cancelled, ydb.DeadlineExceed, asyncio.CancelledError]) +def test_cancellation_and_deadline_retry_policy(error, idempotent, retry_cancelled): + retry_settings = ydb.RetrySettings( + idempotent=idempotent, + retry_cancelled=retry_cancelled, + fast_backoff_settings=ydb.BackoffSettings(slot_duration=0), + ) + + result = check_retriable_error(error("cancelled"), retry_settings, 0) + + expected = (error is ydb.ClientCancelled and idempotent) or (error is ydb.Cancelled and retry_cancelled) + assert result.is_retriable is expected + assert result.sleep_timeout_seconds == (0 if expected else None) + + +@pytest.fixture +async def grpc_stream_endpoint(): + def responses(request, context): + finished = threading.Event() + context.add_callback(finished.set) + yield request + if request == b"cancel": + context.abort(grpc.StatusCode.CANCELLED, "server cancelled") + if request == b"deadline": + context.abort(grpc.StatusCode.DEADLINE_EXCEEDED, "deadline exceeded") + assert finished.wait(timeout=5) + + def bidirectional(requests, context): + yield from responses(next(requests), context) + + with ThreadPoolExecutor(max_workers=4) as executor: + server = grpc.server(executor) + server.add_generic_rpc_handlers( + ( + grpc.method_handlers_generic_handler( + "test.Transport", + { + "Query": grpc.unary_stream_rpc_method_handler(responses), + "Topic": grpc.stream_stream_rpc_method_handler(bidirectional), + }, + ), + ) + ) + port = server.add_insecure_port("127.0.0.1:0") + server.start() + try: + yield f"127.0.0.1:{port}" + finally: + assert await asyncio.to_thread(server.stop(0).wait, timeout=5) + + +class _StreamTestStub: + def __init__(self, channel): + self.Query = channel.unary_stream("/test.Transport/Query") + self.Topic = channel.stream_stream("/test.Transport/Topic") + + +@pytest.mark.parametrize("notify", [False, True]) +@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server", "deadline"]) +def test_sync_stream_distinguishes_channel_close_from_cancellation(grpc_stream_endpoint, cancellation, notify): + disconnected = [] + with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + request = {"server": b"cancel", "deadline": b"deadline"}.get(cancellation, b"first") + stream = connection( + request, + _StreamTestStub, + "Query", + on_disconnected=(lambda: disconnected.append(True)) if notify else None, + ) + assert next(stream) == request + + if cancellation in ("call", "call-and-channel"): + stream.cancel() + if cancellation in ("channel", "call-and-channel"): + connection.close() + + error = { + "channel": ydb.ClientCancelled, + "call": ydb.Cancelled, + "call-and-channel": ydb.Cancelled, + "server": ydb.ClientCancelled, + "deadline": ydb.DeadlineExceed, + }[cancellation] + with pytest.raises(error) as exc_info: + next(stream) + assert type(exc_info.value) is error + assert disconnected == ([True] if cancellation in ("channel", "server") and notify else []) + + +@pytest.mark.parametrize("idempotent", [False, True]) +@pytest.mark.parametrize("failure", ["channel", "server", "deadline"]) +def test_sync_retry_handles_transport_cancellation_and_deadline(grpc_stream_endpoint, failure, idempotent): + attempts = [] + first_request = {"server": b"cancel", "deadline": b"deadline"}.get(failure, b"first") + + def operation(): + with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + stream = connection(first_request if not attempts else b"first", _StreamTestStub, "Query") + attempts.append(next(stream)) + if len(attempts) == 1: + if failure == "channel": + connection.close() + next(stream) + stream.cancel() + return attempts[-1] + + retry_settings = ydb.RetrySettings( + idempotent=idempotent, max_retries=1, fast_backoff_settings=ydb.BackoffSettings(slot_duration=0) + ) + if idempotent and failure != "deadline": + assert ydb.retry_operation_sync(operation, retry_settings=retry_settings) == b"first" + assert attempts == [first_request, b"first"] + else: + error = ydb.DeadlineExceed if failure == "deadline" else ydb.ClientCancelled + with pytest.raises(error): + ydb.retry_operation_sync(operation, retry_settings=retry_settings) + assert attempts == [first_request] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["Query", "Topic"]) +@pytest.mark.parametrize("notify", [False, True]) +@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server", "deadline"]) +async def test_async_stream_distinguishes_channel_close_from_cancellation( + grpc_stream_endpoint, method, cancellation, notify +): + disconnected = [] + + async def on_disconnected(): + disconnected.append(True) + + async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + request = {"server": b"cancel", "deadline": b"deadline"}.get(cancellation, b"first") + stream = await connection( + request if method == "Query" else iter([request]), + _StreamTestStub, + method, + on_disconnected=on_disconnected if notify else None, + ) + assert await asyncio.wait_for(stream.__anext__(), timeout=5) == request + + if cancellation in ("call", "call-and-channel"): + stream.cancel() + if cancellation in ("channel", "call-and-channel"): + await connection.close() + + error = { + "channel": ydb.ClientCancelled, + "call": asyncio.CancelledError, + "call-and-channel": asyncio.CancelledError, + "server": ydb.ClientCancelled, + "deadline": ydb.DeadlineExceed, + }[cancellation] + with pytest.raises(error) as exc_info: + await asyncio.wait_for(stream.__anext__(), timeout=5) + assert type(exc_info.value) is error + assert disconnected == ([True] if cancellation in ("channel", "server") and notify else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("close_channel", [False, True]) +async def test_async_stream_preserves_caller_task_cancellation(grpc_stream_endpoint, close_channel): + entered = asyncio.Event() + disconnected = [] + + async def on_disconnected(): + disconnected.append(True) + + async with grpc.aio.insecure_channel(grpc_stream_endpoint) as channel: + response = channel.unary_stream("/test.Transport/Query")(b"first") + assert await asyncio.wait_for(response.__aiter__().__anext__(), timeout=5) == b"first" + + async def observed(): + entered.set() + async for message in response: + yield message + + stream = _SafeAsyncIterator(observed(), "test", on_disconnected) + task = asyncio.create_task(stream.__anext__()) + try: + await asyncio.wait_for(entered.wait(), timeout=5) + task.cancel() + if close_channel: + await channel.close() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=5) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert disconnected == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["Query", "Topic"]) +@pytest.mark.parametrize("idempotent", [False, True]) +@pytest.mark.parametrize("failure", ["channel", "server", "deadline"]) +async def test_async_retry_handles_transport_cancellation_and_deadline( + grpc_stream_endpoint, method, failure, idempotent +): + from ydb.retries import retry_operation_async + + attempts = [] + first_request = {"server": b"cancel", "deadline": b"deadline"}.get(failure, b"first") + + async def operation(): + async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection: + request = first_request if not attempts else b"first" + stream = await connection(request if method == "Query" else iter([request]), _StreamTestStub, method) + attempts.append(await asyncio.wait_for(stream.__anext__(), timeout=5)) + if len(attempts) == 1: + if failure == "channel": + await connection.close() + await asyncio.wait_for(stream.__anext__(), timeout=5) + stream.cancel() + return attempts[-1] + + retry_settings = ydb.RetrySettings( + idempotent=idempotent, max_retries=1, fast_backoff_settings=ydb.BackoffSettings(slot_duration=0) + ) + if idempotent and failure != "deadline": + assert await retry_operation_async(operation, retry_settings=retry_settings) == b"first" + assert attempts == [first_request, b"first"] + else: + error = ydb.DeadlineExceed if failure == "deadline" else ydb.ClientCancelled + with pytest.raises(error): + await retry_operation_async(operation, retry_settings=retry_settings) + assert attempts == [first_request] + + def test_convert_floats_to_embedding_bytes_encodes_float32_values(): assert ydb.convert_floats_to_embedding_bytes([1, -2, 0.5]) == bytes.fromhex("0000803f000000c00000003f01") diff --git a/ydb/aio/connection.py b/ydb/aio/connection.py index 3bc72f92..d05251ab 100644 --- a/ydb/aio/connection.py +++ b/ydb/aio/connection.py @@ -120,8 +120,10 @@ def __init__(self, resp, rpc_state, on_disconnected_callback): self.it = resp.__aiter__() self.rpc_state = rpc_state self.on_disconnected_callback = on_disconnected_callback + self._cancelled = False def cancel(self): + self._cancelled = True self.resp.cancel() return self @@ -129,13 +131,26 @@ def __aiter__(self): return self async def __anext__(self): + next_response = asyncio.ensure_future(self.it.__anext__()) try: - return await self.it.__anext__() + # wait() keeps caller cancellation separate from gRPC cancelling its + # own read when discovery retires the underlying channel. + await asyncio.wait([next_response]) + except asyncio.CancelledError: + next_response.cancel() + await asyncio.gather(next_response, return_exceptions=True) + raise + try: + return next_response.result() + except asyncio.CancelledError: + if self._cancelled: + raise + ydb_error = issues.ClientCancelled("gRPC stream cancelled") except grpc.RpcError as rpc_error: ydb_error = _rpc_error_handler(self.rpc_state, rpc_error, use_unavailable=True) - if _is_disconnect_needed(ydb_error): - await self.on_disconnected_callback() - raise ydb_error + if _is_disconnect_needed(ydb_error) and self.on_disconnected_callback is not None: + await self.on_disconnected_callback() + raise ydb_error def __getattr__(self, item): return getattr(self.resp, item) diff --git a/ydb/connection.py b/ydb/connection.py index 052b563e..00c63c7a 100644 --- a/ydb/connection.py +++ b/ydb/connection.py @@ -99,7 +99,7 @@ def _rpc_error_handler( elif rpc_error.code() == grpc.StatusCode.UNIMPLEMENTED: return issues.Unimplemented("Method or feature is not implemented on server!") elif rpc_error.code() == grpc.StatusCode.CANCELLED: - return issues.Cancelled(rpc_error.details()) + return issues.ClientCancelled(rpc_error.details()) elif rpc_error.code() == grpc.StatusCode.RESOURCE_EXHAUSTED: if "Sent message larger than max" in rpc_error.details(): return issues.BadRequest(rpc_error.details()) @@ -119,6 +119,7 @@ def _is_disconnect_needed(error): ( issues.ConnectionLost, issues.Unavailable, + issues.ClientCancelled, ), ) @@ -379,8 +380,10 @@ def __init__(self, resp, rpc_state, on_disconnected_callback): self.it = resp.__iter__() self.rpc_state = rpc_state self.on_disconnected_callback = on_disconnected_callback + self._cancelled = False def cancel(self): + self._cancelled = True self.resp.cancel() return self @@ -392,7 +395,9 @@ def __next__(self): return self.it.__next__() except grpc.RpcError as rpc_error: ydb_error = _rpc_error_handler(self.rpc_state, rpc_error, use_unavailable=True) - if _is_disconnect_needed(ydb_error): + if self._cancelled and isinstance(ydb_error, issues.ClientCancelled): + ydb_error = issues.Cancelled(rpc_error.details()) + if _is_disconnect_needed(ydb_error) and self.on_disconnected_callback is not None: self.on_disconnected_callback() raise ydb_error diff --git a/ydb/issues.py b/ydb/issues.py index fba0ee0c..f73c3ebb 100644 --- a/ydb/issues.py +++ b/ydb/issues.py @@ -60,6 +60,7 @@ class StatusCode(enum.IntEnum): DEADLINE_EXCEEDED = _TRANSPORT_STATUSES_FIRST + 30 CLIENT_INTERNAL_ERROR = _TRANSPORT_STATUSES_FIRST + 40 UNIMPLEMENTED = _TRANSPORT_STATUSES_FIRST + 50 + CLIENT_CANCELLED = _TRANSPORT_STATUSES_FIRST + 60 UNAUTHENTICATED = _CLIENT_STATUSES_FIRST + 30 SESSION_POOL_EMPTY = _CLIENT_STATUSES_FIRST + 40 @@ -172,6 +173,10 @@ class Cancelled(Error): status: ClassVar[Optional[StatusCode]] = StatusCode.CANCELLED +class ClientCancelled(Cancelled): + status: ClassVar[Optional[StatusCode]] = StatusCode.CLIENT_CANCELLED + + class Undetermined(Error): status: ClassVar[Optional[StatusCode]] = StatusCode.UNDETERMINED