Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
* Distinguish gRPC cancellation as `ClientCancelled` and retry it only for idempotent operations, preserving explicit caller cancellation and non-retriable deadlines

## 3.33.2 ##
* Fix sync `QuerySessionPool` and `QuerySession.create()` hanging for 600 seconds and then failing with a non-retriable `TimeoutError` when a new session's attach stream fails before its first message (e.g. during a node restart); the attach error is now raised immediately, so retriable errors such as `Unavailable` are retried

Expand Down
3 changes: 3 additions & 0 deletions docs/apireference.rst
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,9 @@ Errors
.. autoclass:: ydb.Undetermined
:undoc-members:

.. autoclass:: ydb.ClientCancelled
:undoc-members:

.. autoclass:: ydb.ConnectionError
:undoc-members:

Expand Down
19 changes: 17 additions & 2 deletions docs/errors.rst
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ Exception Hierarchy
├── ydb.AlreadyExists — object already exists (DDL)
├── ydb.NotFound — object not found
├── ydb.SessionExpired — session TTL expired
├── ydb.Cancelled — operation was cancelled
├── ydb.Cancelled — server status or explicit sync stream cancellation
│ └── ydb.ClientCancelled — gRPC call was cancelled
├── ydb.Undetermined — outcome is unknown (e.g. network lost before ack)
├── ydb.Unsupported — operation not supported by this server version
├── ydb.SessionBusy — session is executing another request
Expand All @@ -41,6 +42,13 @@ Exception Hierarchy
│ └── ydb.Unimplemented — server does not support this RPC
└── ydb.DeadlineExceed — client-side deadline exceeded

``ClientCancelled`` has the distinct status ``StatusCode.CLIENT_CANCELLED``. It
inherits from ``Cancelled`` so existing cancellation handlers continue to catch
it, but its retries are controlled by ``idempotent``, not ``retry_cancelled``.
The server's YDB ``CANCELLED`` status remains ``Cancelled`` with
``StatusCode.CANCELLED``. Explicit async stream or caller-task cancellation
continues to raise ``asyncio.CancelledError``.


Catching Errors
---------------
Expand Down Expand Up @@ -100,6 +108,12 @@ Not every error is worth retrying. The SDK classifies errors into three groups:
* :class:`~ydb.SessionPoolEmpty` — all pool sessions busy; wait and retry.
* :class:`~ydb.ConnectionError` / :class:`~ydb.ConnectionLost` — network issues; reconnect and retry.

**Retriable only for idempotent operations (fast backoff):**

* :class:`~ydb.ClientCancelled` — gRPC ``CANCELLED``, including cancellation caused
by a closing channel. Set ``idempotent=True`` in :class:`~ydb.RetrySettings`
to enable retries. ``retry_cancelled=True`` does not override this requirement.

**Retriable only for idempotent operations (slow backoff):**

* :class:`~ydb.Undetermined` — outcome unknown; only safe to retry if the operation is idempotent
Expand All @@ -108,6 +122,7 @@ Not every error is worth retrying. The SDK classifies errors into three groups:

**Never retried:**

* :class:`~ydb.DeadlineExceed` — gRPC ``DEADLINE_EXCEEDED``.
* :class:`~ydb.BadRequest`, :class:`~ydb.Unauthorized`, ``Unauthenticated``,
:class:`~ydb.SchemeError`, :class:`~ydb.AlreadyExists`, ``Unsupported``,
:class:`~ydb.Timeout`, ``PreconditionFailed``, ``ExternalError`` — these indicate
Expand All @@ -126,7 +141,7 @@ All pool methods that perform retries (``execute_with_retries``, ``retry_operati

retry = ydb.RetrySettings(
max_retries=5, # max retry attempts (default: 10)
idempotent=False, # set True to also retry Undetermined errors
idempotent=False, # set True to retry ClientCancelled and Undetermined
retry_cancelled=False, # set True to retry Cancelled errors
)

Expand Down
5 changes: 5 additions & 0 deletions ydb/_errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@


def check_retriable_error(err, retry_settings, attempt):
if isinstance(err, issues.ClientCancelled):
if retry_settings.idempotent:
return ErrorRetryInfo(True, retry_settings.fast_backoff.calc_timeout(attempt))
return ErrorRetryInfo(False, None)

if isinstance(err, issues.Cancelled):
if retry_settings.retry_cancelled:
return ErrorRetryInfo(True, retry_settings.fast_backoff.calc_timeout(attempt))
Expand Down
270 changes: 270 additions & 0 deletions ydb/_utilities_test.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,24 @@
from pathlib import Path, PurePosixPath
import asyncio
from concurrent.futures import ThreadPoolExecutor
import subprocess
import sys
import tarfile
import threading
from types import SimpleNamespace
from unittest.mock import Mock
import zipfile

import grpc
import pytest

import ydb
from ydb.aio.connection import _SafeAsyncIterator
from ydb.aio.connection import Connection as AsyncConnection
from ydb.connection import Connection as SyncConnection
from ydb.connection import _rpc_error_handler

from ydb._errors import check_retriable_error
from ydb._utilities import check_module_exists
from ydb._utilities import get_first_message_with_timeout
from ydb._utilities import x_ydb_sdk_build_info_header
Expand Down Expand Up @@ -50,6 +61,265 @@ def stream():
assert exc_info.value is error


@pytest.mark.parametrize(
"code,error,status",
[
(grpc.StatusCode.CANCELLED, ydb.ClientCancelled, ydb.StatusCode.CLIENT_CANCELLED),
(grpc.StatusCode.DEADLINE_EXCEEDED, ydb.DeadlineExceed, ydb.StatusCode.DEADLINE_EXCEEDED),
],
)
def test_grpc_cancellation_and_deadline_have_distinct_statuses(code, error, status):
rpc_error = Mock(spec=grpc.Call)
rpc_error.code.return_value = code
rpc_error.details.return_value = "transport error"

result = _rpc_error_handler("test", rpc_error)

assert type(result) is error
assert result.status == status
if error is ydb.ClientCancelled:
assert isinstance(result, ydb.Cancelled)


def test_server_cancellation_preserves_server_status():
response = SimpleNamespace(status=ydb.StatusCode.CANCELLED, issues=[])
with pytest.raises(ydb.Cancelled) as exc_info:
ydb.issues._process_response(response)
assert type(exc_info.value) is ydb.Cancelled
assert exc_info.value.status == ydb.StatusCode.CANCELLED


@pytest.mark.parametrize("idempotent", [False, True])
@pytest.mark.parametrize("retry_cancelled", [False, True])
@pytest.mark.parametrize("error", [ydb.ClientCancelled, ydb.Cancelled, ydb.DeadlineExceed, asyncio.CancelledError])
def test_cancellation_and_deadline_retry_policy(error, idempotent, retry_cancelled):
retry_settings = ydb.RetrySettings(
idempotent=idempotent,
retry_cancelled=retry_cancelled,
fast_backoff_settings=ydb.BackoffSettings(slot_duration=0),
)

result = check_retriable_error(error("cancelled"), retry_settings, 0)

expected = (error is ydb.ClientCancelled and idempotent) or (error is ydb.Cancelled and retry_cancelled)
assert result.is_retriable is expected
assert result.sleep_timeout_seconds == (0 if expected else None)


@pytest.fixture
async def grpc_stream_endpoint():
def responses(request, context):
finished = threading.Event()
context.add_callback(finished.set)
yield request
if request == b"cancel":
context.abort(grpc.StatusCode.CANCELLED, "server cancelled")
if request == b"deadline":
context.abort(grpc.StatusCode.DEADLINE_EXCEEDED, "deadline exceeded")
assert finished.wait(timeout=5)

def bidirectional(requests, context):
yield from responses(next(requests), context)

with ThreadPoolExecutor(max_workers=4) as executor:
server = grpc.server(executor)
server.add_generic_rpc_handlers(
(
grpc.method_handlers_generic_handler(
"test.Transport",
{
"Query": grpc.unary_stream_rpc_method_handler(responses),
"Topic": grpc.stream_stream_rpc_method_handler(bidirectional),
},
),
)
)
port = server.add_insecure_port("127.0.0.1:0")
server.start()
try:
yield f"127.0.0.1:{port}"
finally:
assert await asyncio.to_thread(server.stop(0).wait, timeout=5)


class _StreamTestStub:
def __init__(self, channel):
self.Query = channel.unary_stream("/test.Transport/Query")
self.Topic = channel.stream_stream("/test.Transport/Topic")


@pytest.mark.parametrize("notify", [False, True])
@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server", "deadline"])
def test_sync_stream_distinguishes_channel_close_from_cancellation(grpc_stream_endpoint, cancellation, notify):
disconnected = []
with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection:
request = {"server": b"cancel", "deadline": b"deadline"}.get(cancellation, b"first")
stream = connection(
request,
_StreamTestStub,
"Query",
on_disconnected=(lambda: disconnected.append(True)) if notify else None,
)
assert next(stream) == request

if cancellation in ("call", "call-and-channel"):
stream.cancel()
if cancellation in ("channel", "call-and-channel"):
connection.close()

error = {
"channel": ydb.ClientCancelled,
"call": ydb.Cancelled,
"call-and-channel": ydb.Cancelled,
"server": ydb.ClientCancelled,
"deadline": ydb.DeadlineExceed,
}[cancellation]
with pytest.raises(error) as exc_info:
next(stream)
assert type(exc_info.value) is error
assert disconnected == ([True] if cancellation in ("channel", "server") and notify else [])


@pytest.mark.parametrize("idempotent", [False, True])
@pytest.mark.parametrize("failure", ["channel", "server", "deadline"])
def test_sync_retry_handles_transport_cancellation_and_deadline(grpc_stream_endpoint, failure, idempotent):
attempts = []
first_request = {"server": b"cancel", "deadline": b"deadline"}.get(failure, b"first")

def operation():
with SyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection:
stream = connection(first_request if not attempts else b"first", _StreamTestStub, "Query")
attempts.append(next(stream))
if len(attempts) == 1:
if failure == "channel":
connection.close()
next(stream)
stream.cancel()
return attempts[-1]

retry_settings = ydb.RetrySettings(
idempotent=idempotent, max_retries=1, fast_backoff_settings=ydb.BackoffSettings(slot_duration=0)
)
if idempotent and failure != "deadline":
assert ydb.retry_operation_sync(operation, retry_settings=retry_settings) == b"first"
assert attempts == [first_request, b"first"]
else:
error = ydb.DeadlineExceed if failure == "deadline" else ydb.ClientCancelled
with pytest.raises(error):
ydb.retry_operation_sync(operation, retry_settings=retry_settings)
assert attempts == [first_request]


@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["Query", "Topic"])
@pytest.mark.parametrize("notify", [False, True])
@pytest.mark.parametrize("cancellation", ["channel", "call", "call-and-channel", "server", "deadline"])
async def test_async_stream_distinguishes_channel_close_from_cancellation(
grpc_stream_endpoint, method, cancellation, notify
):
disconnected = []

async def on_disconnected():
disconnected.append(True)

async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection:
request = {"server": b"cancel", "deadline": b"deadline"}.get(cancellation, b"first")
stream = await connection(
request if method == "Query" else iter([request]),
_StreamTestStub,
method,
on_disconnected=on_disconnected if notify else None,
)
assert await asyncio.wait_for(stream.__anext__(), timeout=5) == request

if cancellation in ("call", "call-and-channel"):
stream.cancel()
if cancellation in ("channel", "call-and-channel"):
await connection.close()

error = {
"channel": ydb.ClientCancelled,
"call": asyncio.CancelledError,
"call-and-channel": asyncio.CancelledError,
"server": ydb.ClientCancelled,
"deadline": ydb.DeadlineExceed,
}[cancellation]
with pytest.raises(error) as exc_info:
await asyncio.wait_for(stream.__anext__(), timeout=5)
assert type(exc_info.value) is error
assert disconnected == ([True] if cancellation in ("channel", "server") and notify else [])


@pytest.mark.asyncio
@pytest.mark.parametrize("close_channel", [False, True])
async def test_async_stream_preserves_caller_task_cancellation(grpc_stream_endpoint, close_channel):
entered = asyncio.Event()
disconnected = []

async def on_disconnected():
disconnected.append(True)

async with grpc.aio.insecure_channel(grpc_stream_endpoint) as channel:
response = channel.unary_stream("/test.Transport/Query")(b"first")
assert await asyncio.wait_for(response.__aiter__().__anext__(), timeout=5) == b"first"

async def observed():
entered.set()
async for message in response:
yield message

stream = _SafeAsyncIterator(observed(), "test", on_disconnected)
task = asyncio.create_task(stream.__anext__())
try:
await asyncio.wait_for(entered.wait(), timeout=5)
task.cancel()
if close_channel:
await channel.close()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=5)
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
assert disconnected == []


@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["Query", "Topic"])
@pytest.mark.parametrize("idempotent", [False, True])
@pytest.mark.parametrize("failure", ["channel", "server", "deadline"])
async def test_async_retry_handles_transport_cancellation_and_deadline(
grpc_stream_endpoint, method, failure, idempotent
):
from ydb.retries import retry_operation_async

attempts = []
first_request = {"server": b"cancel", "deadline": b"deadline"}.get(failure, b"first")

async def operation():
async with AsyncConnection(grpc_stream_endpoint, ydb.DriverConfig(grpc_stream_endpoint)) as connection:
request = first_request if not attempts else b"first"
stream = await connection(request if method == "Query" else iter([request]), _StreamTestStub, method)
attempts.append(await asyncio.wait_for(stream.__anext__(), timeout=5))
if len(attempts) == 1:
if failure == "channel":
await connection.close()
await asyncio.wait_for(stream.__anext__(), timeout=5)
stream.cancel()
return attempts[-1]

retry_settings = ydb.RetrySettings(
idempotent=idempotent, max_retries=1, fast_backoff_settings=ydb.BackoffSettings(slot_duration=0)
)
if idempotent and failure != "deadline":
assert await retry_operation_async(operation, retry_settings=retry_settings) == b"first"
assert attempts == [first_request, b"first"]
else:
error = ydb.DeadlineExceed if failure == "deadline" else ydb.ClientCancelled
with pytest.raises(error):
await retry_operation_async(operation, retry_settings=retry_settings)
assert attempts == [first_request]


def test_convert_floats_to_embedding_bytes_encodes_float32_values():
assert ydb.convert_floats_to_embedding_bytes([1, -2, 0.5]) == bytes.fromhex("0000803f000000c00000003f01")

Expand Down
Loading
Loading