diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7adf26f..eac08f3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,8 +13,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.12'] - poetry-version: [1.8] + python-version: ['3.9', '3.12', '3.10'] + poetry-version: ['1.8'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a29fede..8be9a22 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] - poetry-version: [1.8] + poetry-version: ['1.8'] steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index 9898b59..97f303c 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,11 @@ import ydb_dbapi connection = ydb_dbapi.connect( host="localhost", port="2136", database="/local" -) # sync connection +) # sync connection async_connection = await ydb_dbapi.async_connect( host="localhost", port="2136", database="/local" -) # async connection +) # async connection ``` For TLS connections, pass PEM certificate contents as `bytes` through @@ -53,6 +53,40 @@ async with async_connection.cursor() as cursor: rows = await cursor.fetchall() ``` +## Stream cursors (server-side cursor) + +By default results are fetched and buffered by the driver as soon as +`execute()` is called. For large result sets pass `stream_results=True` +to `cursor()`: the driver then streams result sets lazily, fetching the +next chunk only when `fetch*()` needs it. + +```python +with connection.cursor(stream_results=True) as cursor: + cursor.execute("SELECT id, val FROM table") + + while True: + row = cursor.fetchone() + if row is None: + break + print(row) +``` + +```python +async with async_connection.cursor(stream_results=True) as cursor: + await cursor.execute("SELECT id, val FROM table") + + rows = await cursor.fetchall() +``` + +Notes: +- `rowcount` is `-1` while the stream is still being consumed and is + reported only after all remaining rows have been fetched or the cursor is + closed. +- In an interactive transaction only one stream cursor may be active at a + time: `commit()`, `rollback()` and executing another query while a stream + cursor is still streaming raise `ProgrammingError`. Fetch all rows or + close the cursor first. + ## Query parameters ### Standard mode (`pyformat=True`) @@ -68,7 +102,9 @@ and infer YDB types from Python values automatically. ```python connection = ydb_dbapi.connect( - host="localhost", port="2136", database="/local", + host="localhost", + port="2136", + database="/local", pyformat=True, ) @@ -127,7 +163,9 @@ with connection.cursor() as cursor: "INSERT INTO events (id, payload) VALUES (%(id)s, %(payload)s)", { "id": ydb.TypedValue(99, ydb.PrimitiveType.Int32), - "payload": ydb.TypedValue('{"key": "value"}', ydb.PrimitiveType.Json), + "payload": ydb.TypedValue( + '{"key": "value"}', ydb.PrimitiveType.Json + ), }, ) ``` @@ -152,7 +190,9 @@ Use `$name` placeholders in the query and supply a `dict` with ```python connection = ydb_dbapi.connect( - host="localhost", port="2136", database="/local", + host="localhost", + port="2136", + database="/local", ) with connection.cursor() as cursor: diff --git a/tests/test_connections.py b/tests/test_connections.py index 79e6a68..9b600ac 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -173,6 +173,101 @@ def _test_cursor_is_reusable_after_commit( connection.cursor().execute_scheme("DROP TABLE test_tx_context") ) + def _test_stream_cursor_fetch(self, connection: dbapi.Connection) -> None: + cur = connection.cursor() + maybe_await(cur.execute_scheme("DROP TABLE IF EXISTS stream_table")) + maybe_await( + cur.execute_scheme( + """ + CREATE TABLE stream_table ( + id Int64 NOT NULL, + val Int64, + PRIMARY KEY(id) + ) + """ + ) + ) + maybe_await( + cur.execute( + "UPSERT INTO stream_table (id, val) " + "VALUES (0,0),(1,1),(2,2),(3,3)" + ) + ) + maybe_await(cur.close()) + + stream = connection.cursor(stream_results=True) + maybe_await(stream.execute("SELECT id FROM stream_table ORDER BY id")) + + assert stream.rowcount == -1 + assert maybe_await(stream.fetchmany(size=3)) == [(0,), (1,), (2,)] + assert stream.rowcount == -1 + assert maybe_await(stream.fetchall()) == [(3,)] + assert stream.rowcount == 4 + + maybe_await(stream.close()) + maybe_await( + connection.cursor().execute_scheme("DROP TABLE stream_table") + ) + + def _test_stream_cursor_interactive_transaction_guard( + self, connection: dbapi.Connection + ) -> None: + cur = connection.cursor() + maybe_await( + cur.execute_scheme("DROP TABLE IF EXISTS stream_guard_table") + ) + maybe_await( + cur.execute_scheme( + """ + CREATE TABLE stream_guard_table ( + id Int64 NOT NULL, + val Int64, + PRIMARY KEY(id) + ) + """ + ) + ) + maybe_await( + cur.execute( + "UPSERT INTO stream_guard_table (id, val) " + "VALUES (0,0),(1,1),(2,2),(3,3)" + ) + ) + maybe_await(cur.close()) + + connection.set_isolation_level(dbapi.IsolationLevel.SERIALIZABLE) + maybe_await(connection.begin()) + + stream = connection.cursor(stream_results=True) + maybe_await( + stream.execute("SELECT id FROM stream_guard_table ORDER BY id") + ) + assert connection._current_cursor is stream + + other = connection.cursor() + with pytest.raises(dbapi.ProgrammingError): + maybe_await(other.execute("SELECT id FROM stream_guard_table")) + + with pytest.raises(dbapi.ProgrammingError): + maybe_await(connection.commit()) + + rows = [] + while True: + row = maybe_await(stream.fetchone()) + if row is None: + break + rows.append(row) + assert len(rows) == 4 + assert connection._current_cursor is None + + maybe_await(connection.rollback()) + + maybe_await(other.close()) + maybe_await(stream.close()) + maybe_await( + connection.cursor().execute_scheme("DROP TABLE stream_guard_table") + ) + def _test_connection(self, connection: dbapi.Connection) -> None: maybe_await(connection.commit()) maybe_await(connection.rollback()) @@ -665,6 +760,14 @@ def test_get_view_names(self, connection: dbapi.Connection) -> None: def test_get_table_names(self, connection: dbapi.Connection) -> None: self._test_get_table_names(connection) + def test_stream_cursor_fetch(self, connection: dbapi.Connection) -> None: + self._test_stream_cursor_fetch(connection) + + def test_stream_cursor_interactive_transaction_guard( + self, connection: dbapi.Connection + ) -> None: + self._test_stream_cursor_interactive_transaction_guard(connection) + class TestAsyncConnection(BaseDBApiTestSuit): @pytest_asyncio.fixture @@ -826,3 +929,18 @@ async def test_get_table_names( self, connection: dbapi.AsyncConnection ) -> None: await greenlet_spawn(self._test_get_table_names, connection) + + @pytest.mark.asyncio + async def test_stream_cursor_fetch( + self, connection: dbapi.AsyncConnection + ) -> None: + await greenlet_spawn(self._test_stream_cursor_fetch, connection) + + @pytest.mark.asyncio + async def test_stream_cursor_interactive_transaction_guard( + self, connection: dbapi.AsyncConnection + ) -> None: + await greenlet_spawn( + self._test_stream_cursor_interactive_transaction_guard, + connection, + ) diff --git a/tests/test_cursors.py b/tests/test_cursors.py index b10de80..611eee4 100644 --- a/tests/test_cursors.py +++ b/tests/test_cursors.py @@ -3,14 +3,19 @@ from collections.abc import AsyncGenerator from collections.abc import Generator from inspect import iscoroutine +from typing import Any +from unittest import mock import pytest import ydb import ydb_dbapi from sqlalchemy.util import await_only from sqlalchemy.util import greenlet_spawn +from typing_extensions import Self from ydb_dbapi import AsyncCursor +from ydb_dbapi import AsyncStreamCursor from ydb_dbapi import Cursor +from ydb_dbapi import StreamCursor from ydb_dbapi.utils import CursorStatus @@ -334,3 +339,365 @@ async def test_cursor_state_after_error( self, async_cursor: AsyncCursor ) -> None: await greenlet_spawn(self._test_cursor_state_after_error, async_cursor) + + +# Mock-based stream cursor tests + + +RESULT_SET_LENGTH = 4 +RESULT_SET_COUNT = 3 + + +def make_result_sets(count: int = 1) -> list[mock.MagicMock]: + result_sets = [] + for _ in range(count): + rs = mock.MagicMock() + rs.rows = [(row_id, row_id) for row_id in range(RESULT_SET_LENGTH)] + rs.columns = [] + result_sets.append(rs) + return result_sets + + +class _SyncStream: + def __init__(self, result_sets: list) -> None: + self._result_sets = iter(result_sets) + self.cancelled = False + + def __iter__(self) -> _SyncStream: + return self + + def __next__(self) -> Any: + return next(self._result_sets) + + def cancel(self) -> None: + self.cancelled = True + + def __enter__(self) -> Self: + return self + + def __exit__(self, *args: object) -> None: + for _ in self: + pass + + +class _AsyncStream: + def __init__(self, result_sets: list) -> None: + self._result_sets = iter(result_sets) + self.cancelled = False + + def __aiter__(self) -> _AsyncStream: + return self + + async def __anext__(self) -> Any: + try: + return next(self._result_sets) + except StopIteration as exc: + raise StopAsyncIteration from exc + + def cancel(self) -> None: + self.cancelled = True + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *args: object) -> None: + async for _ in self: + pass + + +class _FailingSyncStream(_SyncStream): + def __next__(self) -> Any: + raise ydb.issues.GenericError("boom") + + +class _FailingAsyncStream(_AsyncStream): + async def __anext__(self) -> Any: + raise ydb.issues.GenericError("boom") + + +def _make_sync_cursor(result_sets: list) -> StreamCursor: + session = mock.MagicMock() + session.deleted = False + stream = _SyncStream(result_sets) + session.transaction.return_value.execute.return_value = stream + + def _delete() -> None: + session.deleted = True + + session.delete.side_effect = _delete + + pool = mock.MagicMock() + pool.acquire.return_value = session + released: list = [] + pool.release.side_effect = lambda s: released.append(s) + pool.released_sessions = released + pool.session = session + + connection = mock.MagicMock() + connection._tx_context = None + connection._set_current_cursor = mock.MagicMock() + connection._clear_current_cursor = mock.MagicMock() + connection._invalidate_session = mock.MagicMock() + + cursor = StreamCursor( + connection, + pool, + ydb.QuerySerializableReadWrite(), + request_settings=ydb.BaseRequestSettings(), + retry_settings=ydb.RetrySettings(), + ) + cursor._test_pool = pool + cursor._test_session = session + return cursor + + +def _make_async_cursor(result_sets: list) -> AsyncStreamCursor: + session = mock.AsyncMock() + session.deleted = False + stream = _AsyncStream(result_sets) + tx = mock.MagicMock() + tx.execute = mock.AsyncMock(return_value=stream) + session.transaction = mock.MagicMock(return_value=tx) + + async def _delete() -> None: + session.deleted = True + + session.delete.side_effect = _delete + + pool = mock.MagicMock() + pool.acquire = mock.AsyncMock(return_value=session) + released: list = [] + + async def _release(s: Any) -> None: + released.append(s) + + pool.release = mock.AsyncMock(side_effect=_release) + pool.released_sessions = released + pool.session = session + + connection = mock.MagicMock() + connection._tx_context = None + connection._set_current_cursor = mock.MagicMock() + connection._clear_current_cursor = mock.MagicMock() + connection._invalidate_session = mock.AsyncMock() + + cursor = AsyncStreamCursor( + connection, + pool, + ydb.QuerySerializableReadWrite(), + request_settings=ydb.BaseRequestSettings(), + retry_settings=ydb.RetrySettings(), + ) + cursor._test_pool = pool + cursor._test_session = session + return cursor + + +class TestStreamCursor: + def _make_cursor(self, result_sets: list) -> StreamCursor: + return _make_sync_cursor(result_sets) + + def test_stream_cursor_fetch_one(self) -> None: + cursor = self._make_cursor(make_result_sets()) + try: + cursor.execute("SELECT id, val FROM table") + + assert cursor.rowcount == -1 + + for i in range(RESULT_SET_LENGTH): + row = cursor.fetchone() + assert row is not None + assert row[0] == i + + assert cursor.fetchone() is None + assert cursor.rowcount == RESULT_SET_LENGTH + finally: + cursor.close() + + def test_stream_cursor_fetch_many(self) -> None: + cursor = self._make_cursor(make_result_sets(RESULT_SET_COUNT)) + try: + cursor.execute( + """ + SELECT id, val FROM table; + SELECT id, val FROM table1; + SELECT id, val FROM table2; + """ + ) + + assert cursor.rowcount == -1 + + rows = cursor.fetchmany(size=5) + assert len(rows) == 5 + assert cursor.rowcount == -1 + + rows = cursor.fetchmany(size=7) + assert len(rows) == 7 + assert cursor.fetchmany(size=1) == [] + assert cursor.rowcount == RESULT_SET_LENGTH * RESULT_SET_COUNT + finally: + cursor.close() + + def test_stream_cursor_fetch_all(self) -> None: + cursor = self._make_cursor(make_result_sets()) + try: + cursor.execute("SELECT id, val FROM table") + + rows = cursor.fetchall() + assert len(rows) == RESULT_SET_LENGTH + assert cursor.rowcount == RESULT_SET_LENGTH + assert cursor.fetchall() == [] + finally: + cursor.close() + + def test_stream_cursor_releases_session_after_drain(self) -> None: + cursor = self._make_cursor(make_result_sets()) + pool = cursor._session_pool + try: + cursor.execute("SELECT id, val FROM table") + cursor.fetchall() + + assert pool.released_sessions == [pool.session] + assert pool.session.deleted is False + finally: + cursor.close() + + def test_stream_cursor_discards_session_on_close(self) -> None: + cursor = self._make_cursor(make_result_sets()) + pool = cursor._session_pool + cursor.execute("SELECT id, val FROM table") + cursor.close() + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + + def test_stream_cursor_cancels_stream_on_close(self) -> None: + cursor = self._make_cursor(make_result_sets()) + cursor.execute("SELECT id, val FROM table") + stream = cursor._stream + cursor.close() + + assert stream is not None + assert stream.cancelled is True + + def test_stream_cursor_invalidate_owner_session_on_error(self) -> None: + cursor = self._make_cursor(make_result_sets()) + pool = cursor._session_pool + failing_session = mock.MagicMock() + failing_session.deleted = False + failing_stream = _FailingSyncStream(make_result_sets()) + failing_session.transaction.return_value.execute.return_value = ( + failing_stream + ) + + def _delete() -> None: + failing_session.deleted = True + + failing_session.delete.side_effect = _delete + pool.session = failing_session + pool.acquire.return_value = failing_session + + with pytest.raises(ydb_dbapi.Error): + cursor.execute("SELECT id, val FROM table") + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + assert cursor._state == CursorStatus.finished + + +class TestAsyncStreamCursor: + def _make_cursor(self, result_sets: list) -> AsyncStreamCursor: + return _make_async_cursor(result_sets) + + @pytest.mark.asyncio + async def test_stream_cursor_fetch_one(self) -> None: + cursor = self._make_cursor(make_result_sets()) + try: + await cursor.execute("SELECT id, val FROM table") + + assert cursor.rowcount == -1 + + for i in range(RESULT_SET_LENGTH): + row = await cursor.fetchone() + assert row is not None + assert row[0] == i + + assert await cursor.fetchone() is None + assert cursor.rowcount == RESULT_SET_LENGTH + finally: + await cursor.close() + + @pytest.mark.asyncio + async def test_stream_cursor_fetch_many(self) -> None: + cursor = self._make_cursor(make_result_sets(RESULT_SET_COUNT)) + try: + await cursor.execute( + """ + SELECT id, val FROM table; + SELECT id, val FROM table1; + SELECT id, val FROM table2; + """ + ) + + assert cursor.rowcount == -1 + + rows = await cursor.fetchmany(size=5) + assert len(rows) == 5 + assert cursor.rowcount == -1 + + rows = await cursor.fetchmany(size=7) + assert len(rows) == 7 + assert await cursor.fetchmany(size=1) == [] + assert cursor.rowcount == RESULT_SET_LENGTH * RESULT_SET_COUNT + finally: + await cursor.close() + + @pytest.mark.asyncio + async def test_stream_cursor_fetch_all(self) -> None: + cursor = self._make_cursor(make_result_sets()) + try: + await cursor.execute("SELECT id, val FROM table") + + rows = await cursor.fetchall() + assert len(rows) == RESULT_SET_LENGTH + assert cursor.rowcount == RESULT_SET_LENGTH + assert await cursor.fetchall() == [] + finally: + await cursor.close() + + @pytest.mark.asyncio + async def test_stream_cursor_discards_session_on_close(self) -> None: + cursor = self._make_cursor(make_result_sets()) + pool = cursor._session_pool + await cursor.execute("SELECT id, val FROM table") + await cursor.close() + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + + @pytest.mark.asyncio + async def test_stream_cursor_invalidate_owner_session_on_error( + self, + ) -> None: + cursor = self._make_cursor(make_result_sets()) + pool = cursor._session_pool + failing_session = mock.AsyncMock() + failing_session.deleted = False + failing_stream = _FailingAsyncStream(make_result_sets()) + tx = mock.MagicMock() + tx.execute = mock.AsyncMock(return_value=failing_stream) + failing_session.transaction = mock.MagicMock(return_value=tx) + + async def _delete() -> None: + failing_session.deleted = True + + failing_session.delete.side_effect = _delete + pool.session = failing_session + pool.acquire = mock.AsyncMock(return_value=failing_session) + + with pytest.raises(ydb_dbapi.Error): + await cursor.execute("SELECT id, val FROM table") + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + assert cursor._state == CursorStatus.finished diff --git a/tests/test_stream_memory.py b/tests/test_stream_memory.py new file mode 100644 index 0000000..c1f6bed --- /dev/null +++ b/tests/test_stream_memory.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import tracemalloc +from collections.abc import AsyncGenerator +from collections.abc import Generator +from inspect import iscoroutine +from typing import Union + +import pytest +import pytest_asyncio +import ydb_dbapi as dbapi +from sqlalchemy.util import await_only +from sqlalchemy.util import greenlet_spawn + +ROW_COUNT = 100_000 +BATCH_SIZE = 1_000 +PAYLOAD_SIZE = 256 +TABLE_NAME = "stream_memory_table" + +ConnectionType = Union[dbapi.Connection, dbapi.AsyncConnection] + + +def maybe_await(obj: callable) -> any: + if not iscoroutine(obj): + return obj + return await_only(obj) + + +def _create_table(connection: ConnectionType) -> None: + cursor = connection.cursor() + try: + maybe_await( + cursor.execute_scheme(f"DROP TABLE IF EXISTS {TABLE_NAME}") + ) + maybe_await( + cursor.execute_scheme( + f""" + CREATE TABLE {TABLE_NAME} ( + id Int64 NOT NULL, + payload Utf8, + PRIMARY KEY(id) + ) + """ + ) + ) + payload = "x" * PAYLOAD_SIZE + for start in range(0, ROW_COUNT, BATCH_SIZE): + stop = min(start + BATCH_SIZE, ROW_COUNT) + values = ",".join( + f"({row_id}, '{payload}')" for row_id in range(start, stop) + ) + maybe_await( + cursor.execute( + f"UPSERT INTO {TABLE_NAME} (id, payload) " + f"VALUES {values}" + ) + ) + finally: + maybe_await(cursor.close()) + + +def _drop_table(connection: ConnectionType) -> None: + maybe_await( + connection.cursor().execute_scheme( + f"DROP TABLE IF EXISTS {TABLE_NAME}" + ) + ) + + +def _measure_buffered(connection: ConnectionType) -> tuple[int, int]: + tracemalloc.start() + try: + cursor = connection.cursor() + maybe_await(cursor.execute(f"SELECT id, payload FROM {TABLE_NAME}")) + rows = maybe_await(cursor.fetchall()) + row_count = len(rows) + del rows + maybe_await(cursor.close()) + _, peak = tracemalloc.get_traced_memory() + return row_count, peak + finally: + tracemalloc.stop() + + +def _measure_stream(connection: ConnectionType) -> tuple[int, int]: + tracemalloc.start() + try: + cursor = connection.cursor(stream_results=True) + maybe_await(cursor.execute(f"SELECT id, payload FROM {TABLE_NAME}")) + row_count = 0 + while maybe_await(cursor.fetchone()) is not None: + row_count += 1 + assert cursor.rowcount == ROW_COUNT + maybe_await(cursor.close()) + _, peak = tracemalloc.get_traced_memory() + return row_count, peak + finally: + tracemalloc.stop() + + +def _run_stream_memory_test(connection: ConnectionType) -> None: + _create_table(connection) + try: + buffered_rows, buffered_peak = _measure_buffered(connection) + stream_rows, stream_peak = _measure_stream(connection) + + print( # noqa: T201 + f"\nBUFFERED peak: {buffered_peak / 1024 / 1024:.1f} MiB " + f"({buffered_rows} rows) | STREAM peak: " + f"{stream_peak / 1024 / 1024:.1f} MiB ({stream_rows} rows)" + ) + + assert buffered_rows == ROW_COUNT + assert stream_rows == ROW_COUNT + assert buffered_peak > 10 * 1024 * 1024 + assert stream_peak * 3 < buffered_peak + finally: + _drop_table(connection) + + +@pytest.fixture +def connection( + connection_kwargs: dict, +) -> Generator[dbapi.Connection]: + conn = dbapi.connect(**connection_kwargs) + try: + yield conn + finally: + conn.close() + + +@pytest_asyncio.fixture +async def async_connection( + connection_kwargs: dict, +) -> AsyncGenerator[dbapi.AsyncConnection]: + def connect() -> dbapi.AsyncConnection: + return maybe_await(dbapi.async_connect(**connection_kwargs)) + + conn = await greenlet_spawn(connect) + try: + yield conn + finally: + + def close() -> None: + maybe_await(conn.close()) + + await greenlet_spawn(close) + + +def test_stream_cursor_keeps_memory_bounded( + connection: dbapi.Connection, +) -> None: + _run_stream_memory_test(connection) + + +@pytest.mark.asyncio +async def test_async_stream_cursor_keeps_memory_bounded( + async_connection: dbapi.AsyncConnection, +) -> None: + await greenlet_spawn(_run_stream_memory_test, async_connection) diff --git a/ydb_dbapi/__init__.py b/ydb_dbapi/__init__.py index 1d2f2dd..bd79565 100644 --- a/ydb_dbapi/__init__.py +++ b/ydb_dbapi/__init__.py @@ -5,7 +5,9 @@ from .connections import connect from .constants import * from .cursors import AsyncCursor +from .cursors import AsyncStreamCursor from .cursors import Cursor +from .cursors import StreamCursor from .errors import * from .version import VERSION diff --git a/ydb_dbapi/connections.py b/ydb_dbapi/connections.py index c08ea23..3e7fe8b 100644 --- a/ydb_dbapi/connections.py +++ b/ydb_dbapi/connections.py @@ -17,11 +17,14 @@ from ydb.retries import retry_operation_sync from .cursors import AsyncCursor +from .cursors import AsyncStreamCursor from .cursors import Cursor +from .cursors import StreamCursor from .errors import InterfaceError from .errors import InternalError from .errors import NotSupportedError from .errors import ProgrammingError +from .utils import CursorStatus from .utils import handle_ydb_errors from .utils import maybe_get_current_trace_id from .utils import prepare_credentials @@ -233,6 +236,7 @@ class Connection(BaseConnection): _driver_cls = ydb.Driver _pool_cls = ydb.QuerySessionPool _cursor_cls = Cursor + _stream_cursor_cls = StreamCursor _driver: ydb.Driver _session_pool: SessionPool _session: ydb.QuerySession | None @@ -267,7 +271,8 @@ def __init__( pyformat=pyformat, **kwargs, ) - self._current_cursor: Cursor | None = None + self._current_cursor: Cursor | StreamCursor | None = None + self._active_stream_cursors: set[StreamCursor] = set() def _create_session_pool( self, driver_config: ydb.DriverConfig @@ -275,8 +280,11 @@ def _create_session_pool( driver = self._driver_cls(driver_config) return self._pool_cls(driver, size=5) - def cursor(self) -> Cursor: - return self._cursor_cls( + def cursor(self, stream_results: bool = False) -> Cursor | StreamCursor: + cursor_cls = ( + self._stream_cursor_cls if stream_results else self._cursor_cls + ) + return cursor_cls( connection=self, session_pool=self._session_pool, tx_mode=self._tx_mode, @@ -301,13 +309,21 @@ def wait_ready(self, timeout: int = 10) -> None: @handle_ydb_errors def begin(self) -> None: + self._raise_if_current_cursor_running() self._tx_context = None if self.interactive_transaction: self._session = self._session_pool.acquire() self._tx_context = self._session.transaction(self._tx_mode) - @handle_ydb_errors def commit(self) -> None: + self._raise_if_current_cursor_running() + try: + self._commit() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + def _commit(self) -> None: if self._tx_context: settings = self._get_request_settings() self._tx_context.commit(settings=settings) @@ -316,8 +332,15 @@ def commit(self) -> None: self._session_pool.release(self._session) self._session = None - @handle_ydb_errors def rollback(self) -> None: + self._raise_if_current_cursor_running() + try: + self._rollback() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + def _rollback(self) -> None: if self._tx_context: settings = self._get_request_settings() self._tx_context.rollback(settings=settings) @@ -328,6 +351,11 @@ def rollback(self) -> None: @handle_ydb_errors def close(self) -> None: + for cursor in list(self._active_stream_cursors): + cursor.close() + self._active_stream_cursors.clear() + if self._current_cursor is not None: + self._current_cursor.close() try: self.rollback() finally: @@ -436,17 +464,44 @@ def bulk_upsert( ) def _invalidate_session(self) -> None: + self._clear_current_cursor() if self._tx_context: self._tx_context = None if self._session: self._session_pool.release(self._session) self._session = None + def _set_current_cursor(self, cursor: Cursor | StreamCursor) -> None: + self._current_cursor = cursor + + def _clear_current_cursor(self, cursor: Cursor | None = None) -> None: + if cursor is None or self._current_cursor is cursor: + self._current_cursor = None + + def _register_stream_cursor(self, cursor: StreamCursor) -> None: + self._active_stream_cursors.add(cursor) + + def _unregister_stream_cursor(self, cursor: StreamCursor) -> None: + self._active_stream_cursors.discard(cursor) + + def _raise_if_current_cursor_running( + self, cursor: Cursor | None = None + ) -> None: + if self._current_cursor is None or self._current_cursor is cursor: + return + if self._current_cursor._state != CursorStatus.running: + return + raise ProgrammingError( + "Could not perform operation: a server-side cursor is still " + "streaming results for the current transaction session." + ) + class AsyncConnection(BaseConnection): _driver_cls = ydb.aio.Driver _pool_cls = ydb.aio.QuerySessionPool _cursor_cls = AsyncCursor + _stream_cursor_cls = AsyncStreamCursor _driver: ydb.aio.Driver _session_pool: AsyncSessionPool _session: ydb.aio.QuerySession | None @@ -481,7 +536,8 @@ def __init__( pyformat=pyformat, **kwargs, ) - self._current_cursor: AsyncCursor | None = None + self._current_cursor: AsyncStreamCursor | None = None + self._active_stream_cursors: set[AsyncStreamCursor] = set() def _create_session_pool( self, driver_config: ydb.DriverConfig @@ -489,8 +545,13 @@ def _create_session_pool( driver = self._driver_cls(driver_config) return self._pool_cls(driver, size=5) - def cursor(self) -> AsyncCursor: - return self._cursor_cls( + def cursor( + self, stream_results: bool = False + ) -> AsyncCursor | AsyncStreamCursor: + cursor_cls = ( + self._stream_cursor_cls if stream_results else self._cursor_cls + ) + return cursor_cls( connection=self, session_pool=self._session_pool, tx_mode=self._tx_mode, @@ -515,13 +576,21 @@ async def wait_ready(self, timeout: int = 10) -> None: @handle_ydb_errors async def begin(self) -> None: + self._raise_if_current_cursor_running() self._tx_context = None if self.interactive_transaction: self._session = await self._session_pool.acquire() self._tx_context = self._session.transaction(self._tx_mode) - @handle_ydb_errors async def commit(self) -> None: + self._raise_if_current_cursor_running() + try: + await self._commit() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + async def _commit(self) -> None: if self._tx_context: settings = self._get_request_settings() await self._tx_context.commit(settings=settings) @@ -530,8 +599,15 @@ async def commit(self) -> None: await self._session_pool.release(self._session) self._session = None - @handle_ydb_errors async def rollback(self) -> None: + self._raise_if_current_cursor_running() + try: + await self._rollback() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + async def _rollback(self) -> None: if self._tx_context: settings = self._get_request_settings() await self._tx_context.rollback(settings=settings) @@ -542,6 +618,11 @@ async def rollback(self) -> None: @handle_ydb_errors async def close(self) -> None: + for cursor in list(self._active_stream_cursors): + await cursor.close() + self._active_stream_cursors.clear() + if self._current_cursor is not None: + await self._current_cursor.close() try: await self.rollback() finally: @@ -654,12 +735,40 @@ async def bulk_upsert( ) async def _invalidate_session(self) -> None: + self._clear_current_cursor() if self._tx_context: self._tx_context = None if self._session: await self._session_pool.release(self._session) self._session = None + def _set_current_cursor(self, cursor: AsyncStreamCursor) -> None: + self._current_cursor = cursor + + def _clear_current_cursor( + self, cursor: AsyncStreamCursor | None = None + ) -> None: + if cursor is None or self._current_cursor is cursor: + self._current_cursor = None + + def _register_stream_cursor(self, cursor: AsyncStreamCursor) -> None: + self._active_stream_cursors.add(cursor) + + def _unregister_stream_cursor(self, cursor: AsyncStreamCursor) -> None: + self._active_stream_cursors.discard(cursor) + + def _raise_if_current_cursor_running( + self, cursor: AsyncStreamCursor | None = None + ) -> None: + if self._current_cursor is None or self._current_cursor is cursor: + return + if self._current_cursor._state != CursorStatus.running: + return + raise ProgrammingError( + "Could not perform operation: a server-side cursor is still " + "streaming results for the current transaction session." + ) + def connect( host: str = "", diff --git a/ydb_dbapi/cursors.py b/ydb_dbapi/cursors.py index c9bfb32..dffa8c4 100644 --- a/ydb_dbapi/cursors.py +++ b/ydb_dbapi/cursors.py @@ -13,6 +13,10 @@ from typing import Union import ydb +from ydb.aio.query.base import AsyncResponseContextIterator +from ydb.query.base import SyncResponseContextIterator +from ydb.retries import retry_operation_async +from ydb.retries import retry_operation_sync from .errors import DatabaseError from .errors import InterfaceError @@ -66,7 +70,7 @@ async def awrapper( return await func(self, *args, **kwargs) except ydb.Error: self._state = CursorStatus.finished - await self._connection._invalidate_session() + await self._invalidate_active_session() raise return awrapper @@ -77,7 +81,7 @@ def wrapper(self: Cursor, *args: tuple, **kwargs: dict) -> Any: return func(self, *args, **kwargs) except ydb.Error: self._state = CursorStatus.finished - self._connection._invalidate_session() + self._invalidate_active_session() raise return wrapper @@ -258,6 +262,9 @@ def _get_request_settings(self) -> ydb.BaseRequestSettings: return settings + def _invalidate_active_session(self) -> None: + self._connection._invalidate_session() + def _materialize( self, stream: Iterator[ydb.convert.ResultSet] ) -> list[ydb.convert.ResultSet]: @@ -336,6 +343,9 @@ def execute_scheme( parameters: ParametersType | None = None, ) -> None: self._raise_if_closed() + self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) ydb_parameters = _as_ydb_parameters(parameters) @@ -354,6 +364,8 @@ def execute( ) -> None: self._raise_if_closed() self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) @@ -404,6 +416,271 @@ def __exit__( self.close() +class StreamCursor(Cursor): + def __init__( + self, + connection: Connection, + session_pool: ydb.QuerySessionPool, + tx_mode: ydb.BaseQueryTxMode, + request_settings: ydb.BaseRequestSettings, + retry_settings: ydb.RetrySettings, + table_path_prefix: str = "", + pyformat: bool = False, + ) -> None: + super().__init__( + connection=connection, + session_pool=session_pool, + tx_mode=tx_mode, + request_settings=request_settings, + retry_settings=retry_settings, + table_path_prefix=table_path_prefix, + pyformat=pyformat, + ) + self._stream: SyncResponseContextIterator | None = None + self._session_owner: ydb.QuerySession | None = None + self._rowcount_accumulator: int = 0 + + def _invalidate_active_session(self) -> None: + self._clear_current_cursor() + self._stream = None + if self._session_owner is None: + self._connection._invalidate_session() + return + + session = self._session_owner + self._session_owner = None + self._stream = None + try: + session.delete() + finally: + self._session_pool.release(session) + + def _clear_current_cursor(self) -> None: + self._connection._clear_current_cursor(self) + self._connection._unregister_stream_cursor(self) + + def _register_current_cursor(self) -> None: + if self._tx_context is not None: + self._connection._set_current_cursor(self) + self._connection._register_stream_cursor(self) + + def _release_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + self._session_pool.release(session) + + def _discard_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + try: + session.delete() + finally: + self._session_pool.release(session) + + def _finish_stream(self) -> None: + self._stream = None + self._release_owned_session() + self._clear_current_cursor() + self._finish_query() + + def _reset_result(self) -> None: + super()._reset_result() + self._rowcount_accumulator = 0 + + def _update_result_set( + self, + result_set: ydb.convert.ResultSet, + replace_current: bool = True, + ) -> None: + # A streaming cursor reports the total number of fetched rows only + # after the stream is exhausted, so here the count is accumulated + # and the final value is set in _finish_query(). + self._update_description(result_set) + self._rowcount_accumulator += len(result_set.rows) + + new_rows_iter = self._rows_iterable(result_set) + + if self._rows is None or replace_current: + self._rows = new_rows_iter + else: + self._rows = itertools.chain(self._rows, new_rows_iter) + + def _finalize_rowcount(self) -> None: + self._rows_count = self._rowcount_accumulator + + def _finish_query(self) -> None: + self._finalize_rowcount() + super()._finish_query() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + def _load_next_result_set(self) -> bool: + if self._stream is None: + return False + + try: + result_set = next(self._stream) + except StopIteration: + self._finish_stream() + return False + + self._update_result_set(result_set, replace_current=False) + return True + + def _prime_stream(self) -> None: + if ( + not self._load_next_result_set() + and self._state == CursorStatus.running + ): + self._finish_stream() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + def _execute_session_query_stream( + self, + query: str, + parameters: dict | None = None, + ) -> SyncResponseContextIterator: + settings = self._get_request_settings() + + def callee() -> SyncResponseContextIterator: + acquire_timeout = getattr( + self._retry_settings, + "max_session_acquire_timeout", + None, + ) + session = self._session_pool.acquire(timeout=acquire_timeout) + try: + stream = session.transaction(self._tx_mode).execute( + query=query, + parameters=parameters, + commit_tx=True, + settings=settings, + ) + except Exception: + self._session_pool.release(session) + raise + + self._session_owner = session + return stream + + return retry_operation_sync( + callee, + self._retry_settings, + ) + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + def _execute_transactional_query_stream( + self, + tx_context: ydb.QueryTxContext, + query: str, + parameters: dict | None = None, + ) -> SyncResponseContextIterator: + settings = self._get_request_settings() + return tx_context.execute( + query=query, + parameters=parameters, + commit_tx=False, + settings=settings, + ) + + def execute( + self, + query: str, + parameters: ParametersType | None = None, + ) -> None: + self._raise_if_closed() + self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running(self) + + query = self._append_table_path_prefix(query) + + if self._pyformat and parameters is not None: + query, parameters = convert_query_parameters(query, parameters) + + ydb_parameters = _as_ydb_parameters(parameters) + self._begin_query() + + if self._tx_context is not None: + self._stream = self._execute_transactional_query_stream( + tx_context=self._tx_context, + query=query, + parameters=ydb_parameters, + ) + self._register_current_cursor() + else: + self._stream = self._execute_session_query_stream( + query=query, + parameters=ydb_parameters, + ) + self._register_current_cursor() + + self._prime_stream() + + def fetchone(self) -> tuple | None: + self._raise_if_closed() + + while True: + row = self._fetchone_from_buffer() + if row is not None: + return row + if not self._load_next_result_set(): + return None + + def fetchmany(self, size: int | None = None) -> list: + self._raise_if_closed() + result: list[tuple] = [] + target_size = size or self.arraysize + + while len(result) < target_size: + row = self.fetchone() + if row is None: + break + result.append(row) + + return result + + def fetchall(self) -> list: + self._raise_if_closed() + result = list(self._fetchall_from_buffer()) + + while self._load_next_result_set(): + result.extend(self._fetchall_from_buffer()) + + return result + + def close(self) -> None: + if self._state == CursorStatus.closed: + return + + if self._state == CursorStatus.running: + if self._session_owner is not None and self._stream is not None: + self._stream.cancel() + self._finalize_rowcount() + self._stream = None + self._discard_owned_session() + elif self._stream is not None: + with self._stream: + pass + self._finish_stream() + else: + self._release_owned_session() + self._clear_current_cursor() + + self._stream = None + self._session_owner = None + self._clear_current_cursor() + self._state = CursorStatus.closed + + class AsyncCursor(BufferedCursor): def __init__( self, @@ -449,6 +726,9 @@ def _get_request_settings(self) -> ydb.BaseRequestSettings: return settings + async def _invalidate_active_session(self) -> None: + await self._connection._invalidate_session() + async def _materialize( self, stream: AsyncIterator[ydb.convert.ResultSet] ) -> list[ydb.convert.ResultSet]: @@ -527,6 +807,9 @@ async def execute_scheme( parameters: ParametersType | None = None, ) -> None: self._raise_if_closed() + self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) ydb_parameters = _as_ydb_parameters(parameters) @@ -545,6 +828,8 @@ async def execute( ) -> None: self._raise_if_closed() self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) @@ -593,3 +878,353 @@ async def __aexit__( tb: object, ) -> None: self.close() + + +class AsyncStreamCursor(BufferedCursor): + def __init__( + self, + connection: AsyncConnection, + session_pool: ydb.aio.QuerySessionPool, + tx_mode: ydb.BaseQueryTxMode, + request_settings: ydb.BaseRequestSettings, + retry_settings: ydb.RetrySettings, + table_path_prefix: str = "", + pyformat: bool = False, + ) -> None: + super().__init__() + self._connection = connection + self._session_pool = session_pool + self._tx_mode = tx_mode + self._request_settings = request_settings + self._retry_settings = retry_settings + self._table_path_prefix = table_path_prefix + self._pyformat = pyformat + self._stream: AsyncResponseContextIterator | None = None + self._session_owner: ydb.aio.QuerySession | None = None + self._rowcount_accumulator: int = 0 + + @property + def _tx_context(self) -> ydb.aio.QueryTxContext | None: + # Read the transaction from the connection on every access: a cursor + # must not pin the transaction that was open when it was created. + return self._connection._tx_context + + def _get_request_settings(self) -> ydb.BaseRequestSettings: + settings = self._request_settings.make_copy() + + if self._request_settings.trace_id is None: + settings = settings.with_trace_id(maybe_get_current_trace_id()) + + return settings + + async def _materialize( + self, stream: AsyncIterator[ydb.convert.ResultSet] + ) -> list[ydb.convert.ResultSet]: + return [result_set async for result_set in stream] + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _execute_generic_query( + self, query: str, parameters: ParametersType | None = None + ) -> list[ydb.convert.ResultSet]: + settings = self._get_request_settings() + ydb_parameters = _as_ydb_parameters(parameters) + + async def callee( + session: ydb.aio.QuerySession, + ) -> list[ydb.convert.ResultSet]: + return await self._materialize( + await session.execute( + query=query, + parameters=ydb_parameters, + settings=settings, + ) + ) + + return await self._session_pool.retry_operation_async( + callee, + retry_settings=self._retry_settings, + ) + + async def execute_scheme( + self, + query: str, + parameters: ParametersType | None = None, + ) -> None: + self._raise_if_closed() + self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() + + query = self._append_table_path_prefix(query) + + if self._pyformat and parameters is not None: + query, parameters = convert_query_parameters(query, parameters) + + ydb_parameters = _as_ydb_parameters(parameters) + self._begin_query() + + result_list = await self._execute_generic_query( + query=query, parameters=ydb_parameters + ) + self._fill_buffer(result_list) + self._finish_query() + + async def _invalidate_active_session(self) -> None: + self._clear_current_cursor() + self._stream = None + if self._session_owner is None: + await self._connection._invalidate_session() + return + + session = self._session_owner + self._session_owner = None + try: + await session.delete() + finally: + await self._session_pool.release(session) + + def _clear_current_cursor(self) -> None: + self._connection._clear_current_cursor(self) + self._connection._unregister_stream_cursor(self) + + def _register_current_cursor(self) -> None: + if self._tx_context is not None: + self._connection._set_current_cursor(self) + self._connection._register_stream_cursor(self) + + async def _release_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + await self._session_pool.release(session) + + async def _discard_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + try: + await session.delete() + finally: + await self._session_pool.release(session) + + async def _finish_stream(self) -> None: + self._stream = None + await self._release_owned_session() + self._clear_current_cursor() + self._finish_query() + + def _reset_result(self) -> None: + super()._reset_result() + self._rowcount_accumulator = 0 + + def _update_result_set( + self, + result_set: ydb.convert.ResultSet, + replace_current: bool = True, + ) -> None: + # A streaming cursor reports the total number of fetched rows only + # after the stream is exhausted, so here the count is accumulated + # and the final value is set in _finish_query(). + self._update_description(result_set) + self._rowcount_accumulator += len(result_set.rows) + + new_rows_iter = self._rows_iterable(result_set) + + if self._rows is None or replace_current: + self._rows = new_rows_iter + else: + self._rows = itertools.chain(self._rows, new_rows_iter) + + def _finalize_rowcount(self) -> None: + self._rows_count = self._rowcount_accumulator + + def _finish_query(self) -> None: + self._finalize_rowcount() + super()._finish_query() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _load_next_result_set(self) -> bool: + if self._stream is None: + return False + + try: + result_set = await self._stream.__anext__() + except StopAsyncIteration: + await self._finish_stream() + return False + + self._update_result_set(result_set, replace_current=False) + return True + + async def _prime_stream(self) -> None: + if ( + not await self._load_next_result_set() + and self._state == CursorStatus.running + ): + await self._finish_stream() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _execute_session_query_stream( + self, + query: str, + parameters: dict | None = None, + ) -> AsyncResponseContextIterator: + settings = self._get_request_settings() + + async def callee() -> AsyncResponseContextIterator: + acquire_timeout = getattr( + self._retry_settings, + "max_session_acquire_timeout", + None, + ) + session = await self._session_pool.acquire(timeout=acquire_timeout) + try: + stream = await session.transaction(self._tx_mode).execute( + query=query, + parameters=parameters, + commit_tx=True, + settings=settings, + ) + except Exception: + await self._session_pool.release(session) + raise + + self._session_owner = session + return stream + + return await retry_operation_async( + callee, + self._retry_settings, + ) + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _execute_transactional_query_stream( + self, + tx_context: ydb.aio.QueryTxContext, + query: str, + parameters: dict | None = None, + ) -> AsyncResponseContextIterator: + settings = self._get_request_settings() + return await tx_context.execute( + query=query, + parameters=parameters, + commit_tx=False, + settings=settings, + ) + + async def executemany( + self, query: str, seq_of_parameters: Sequence[ParametersType] + ) -> None: + for parameters in seq_of_parameters: + await self.execute(query, parameters) + + async def execute( + self, + query: str, + parameters: ParametersType | None = None, + ) -> None: + self._raise_if_closed() + self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running(self) + + query = self._append_table_path_prefix(query) + + if self._pyformat and parameters is not None: + query, parameters = convert_query_parameters(query, parameters) + + ydb_parameters = _as_ydb_parameters(parameters) + self._begin_query() + + if self._tx_context is not None: + self._stream = await self._execute_transactional_query_stream( + tx_context=self._tx_context, + query=query, + parameters=ydb_parameters, + ) + self._register_current_cursor() + else: + self._stream = await self._execute_session_query_stream( + query=query, + parameters=ydb_parameters, + ) + self._register_current_cursor() + + await self._prime_stream() + + async def fetchone(self) -> tuple | None: + self._raise_if_closed() + + while True: + row = self._fetchone_from_buffer() + if row is not None: + return row + if not await self._load_next_result_set(): + return None + + async def fetchmany(self, size: int | None = None) -> list: + self._raise_if_closed() + result: list[tuple] = [] + target_size = size or self.arraysize + + while len(result) < target_size: + row = await self.fetchone() + if row is None: + break + result.append(row) + + return result + + async def fetchall(self) -> list: + self._raise_if_closed() + result = list(self._fetchall_from_buffer()) + + while await self._load_next_result_set(): + result.extend(self._fetchall_from_buffer()) + + return result + + async def nextset(self) -> bool: + return False + + async def close(self) -> None: + if self._state == CursorStatus.closed: + return + + if self._state == CursorStatus.running: + if self._session_owner is not None and self._stream is not None: + self._stream.cancel() + self._finalize_rowcount() + self._stream = None + await self._discard_owned_session() + elif self._stream is not None: + async with self._stream: + pass + await self._finish_stream() + else: + await self._release_owned_session() + self._clear_current_cursor() + + self._stream = None + self._session_owner = None + self._clear_current_cursor() + self._state = CursorStatus.closed + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: object, + ) -> None: + await self.close()