From c7bf85bfcf182b8b6e75738b54a630005e1b7985 Mon Sep 17 00:00:00 2001 From: GrigorySmirnovSpb Date: Mon, 26 Oct 2026 22:07:02 +0300 Subject: [PATCH 1/4] WIP: stream cursor --- CHANGELOG.md | 5 + README.md | 50 ++- poetry.lock | 35 +- pyproject.toml | 2 +- tests/test_connections.py | 142 ++++++- tests/test_convert_parameters.py | 4 +- tests/test_cursors.py | 386 +++++++++++++++++++ tests/test_stream_memory.py | 157 ++++++++ ydb_dbapi/__init__.py | 2 + ydb_dbapi/connections.py | 107 +++++- ydb_dbapi/cursors.py | 616 ++++++++++++++++++++++++++++++- ydb_dbapi/utils.py | 6 +- ydb_dbapi/version.py | 2 +- 13 files changed, 1469 insertions(+), 45 deletions(-) create mode 100644 tests/test_stream_memory.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f2f392f..cf8e138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.1.24 ## +* Add stream cursors (server-side cursor): connection.cursor(stream_results=True) +* Guard transactional operations while a stream cursor is streaming +* Sync and async stream cursors with lazy result-set fetching + ## 0.1.23 ## * Read tx context from connection on cursor execute * Release session in close() even if rollback fails diff --git a/README.md b/README.md index 5824376..5dd1445 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,11 @@ import ydb_dbapi connection = ydb_dbapi.connect( host="localhost", port="2136", database="/local" -) # sync connection +) # sync connection async_connection = await ydb_dbapi.async_connect( host="localhost", port="2136", database="/local" -) # async connection +) # async connection ``` Usage of connection: @@ -48,6 +48,40 @@ async with async_connection.cursor() as cursor: rows = await cursor.fetchall() ``` +## Stream cursors (server-side cursor) + +By default results are fetched and buffered by the driver as soon as +`execute()` is called. For large result sets pass `stream_results=True` +to `cursor()`: the driver then streams result sets lazily, fetching the +next chunk only when `fetch*()` needs it. + +```python +with connection.cursor(stream_results=True) as cursor: + cursor.execute("SELECT id, val FROM table") + + while True: + row = cursor.fetchone() + if row is None: + break + print(row) +``` + +```python +async with async_connection.cursor(stream_results=True) as cursor: + await cursor.execute("SELECT id, val FROM table") + + rows = await cursor.fetchall() +``` + +Notes: +- `rowcount` is `-1` while the stream is still being consumed and is + reported only after all remaining rows have been fetched or the cursor is + closed. +- In an interactive transaction only one stream cursor may be active at a + time: `commit()`, `rollback()` and executing another query while a stream + cursor is still streaming raise `ProgrammingError`. Fetch all rows or + close the cursor first. + ## Query parameters ### Standard mode (`pyformat=True`) @@ -63,7 +97,9 @@ and infer YDB types from Python values automatically. ```python connection = ydb_dbapi.connect( - host="localhost", port="2136", database="/local", + host="localhost", + port="2136", + database="/local", pyformat=True, ) @@ -122,7 +158,9 @@ with connection.cursor() as cursor: "INSERT INTO events (id, payload) VALUES (%(id)s, %(payload)s)", { "id": ydb.TypedValue(99, ydb.PrimitiveType.Int32), - "payload": ydb.TypedValue('{"key": "value"}', ydb.PrimitiveType.Json), + "payload": ydb.TypedValue( + '{"key": "value"}', ydb.PrimitiveType.Json + ), }, ) ``` @@ -143,7 +181,9 @@ in the query and supply a `dict` with `$`-prefixed keys: ```python connection = ydb_dbapi.connect( - host="localhost", port="2136", database="/local", + host="localhost", + port="2136", + database="/local", ) with connection.cursor() as cursor: diff --git a/poetry.lock b/poetry.lock index d1752b3..9b3db3f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -2493,8 +2493,8 @@ files = [ [package.dependencies] distlib = ">=0.3.7,<1" filelock = [ - {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""}, {version = ">=3.16.1,<=3.19.1", markers = "python_version < \"3.10\""}, + {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""}, ] platformdirs = ">=3.9.1,<5" python-discovery = ">=1" @@ -2760,14 +2760,38 @@ propcache = ">=0.2.1" [[package]] name = "ydb" -version = "3.26.7" +version = "3.29.7" description = "YDB Python SDK" optional = false python-versions = "*" groups = ["main"] +markers = "python_version < \"3.10\"" +files = [ + {file = "ydb-3.29.7-py2.py3-none-any.whl", hash = "sha256:2b7080721dc1e194b3872a12e61a4f7c67f4930d712a92bef379ba2af9b17efc"}, + {file = "ydb-3.29.7.tar.gz", hash = "sha256:4c113d840de79551b4eb9d01ecbdc357c138f5a3c1df76439dfe72f3253055f5"}, +] + +[package.dependencies] +aiohttp = "<4" +grpcio = ">=1.42.0" +packaging = "*" +protobuf = ">=3.13.0,<7.0.0" + +[package.extras] +opentelemetry = ["opentelemetry-api (>=1.0.0)"] +yc = ["yandexcloud"] + +[[package]] +name = "ydb" +version = "3.33.2" +description = "YDB Python SDK" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "ydb-3.26.7-py2.py3-none-any.whl", hash = "sha256:14f7e47ced588f449ac6d7b63de8ab7bf779e6df08d7724ebb62afaaab27024c"}, - {file = "ydb-3.26.7.tar.gz", hash = "sha256:2e4fc1e7be9e225ea7bd7f5a97abbce648c3d89a16d549ad21ddef16e3f00dfd"}, + {file = "ydb-3.33.2-py3-none-any.whl", hash = "sha256:f69358a7452b6967cbd84160c5b0422e2996d9f37e6923b471a0a9b314599555"}, + {file = "ydb-3.33.2.tar.gz", hash = "sha256:927efa83bc7fec8366eebc01a100eb07389bacb4e9a1827cd801606836c66f86"}, ] [package.dependencies] @@ -2777,6 +2801,7 @@ packaging = "*" protobuf = ">=3.13.0,<7.0.0" [package.extras] +opentelemetry = ["opentelemetry-api (>=1.0.0)"] yc = ["yandexcloud"] [metadata] diff --git a/pyproject.toml b/pyproject.toml index f1345b1..5de90ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ydb-dbapi" -version = "0.1.23" # AUTOVERSION +version = "0.1.24" # AUTOVERSION description = "YDB Python DBAPI which complies with PEP 249" authors = ["Yandex LLC "] readme = "README.md" diff --git a/tests/test_connections.py b/tests/test_connections.py index e70664a..bddb67e 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -65,7 +65,6 @@ def _test_commit_rollback_after_begin( maybe_await(connection.begin()) maybe_await(connection.rollback()) - def _test_close_releases_session_after_rollback_error( self, connection: dbapi.Connection, @@ -174,6 +173,101 @@ def _test_cursor_is_reusable_after_commit( connection.cursor().execute_scheme("DROP TABLE test_tx_context") ) + def _test_stream_cursor_fetch(self, connection: dbapi.Connection) -> None: + cur = connection.cursor() + maybe_await(cur.execute_scheme("DROP TABLE IF EXISTS stream_table")) + maybe_await( + cur.execute_scheme( + """ + CREATE TABLE stream_table ( + id Int64 NOT NULL, + val Int64, + PRIMARY KEY(id) + ) + """ + ) + ) + maybe_await( + cur.execute( + "UPSERT INTO stream_table (id, val) " + "VALUES (0,0),(1,1),(2,2),(3,3)" + ) + ) + maybe_await(cur.close()) + + stream = connection.cursor(stream_results=True) + maybe_await(stream.execute("SELECT id FROM stream_table ORDER BY id")) + + assert stream.rowcount == -1 + assert maybe_await(stream.fetchmany(size=3)) == [(0,), (1,), (2,)] + assert stream.rowcount == -1 + assert maybe_await(stream.fetchall()) == [(3,)] + assert stream.rowcount == 4 + + maybe_await(stream.close()) + maybe_await( + connection.cursor().execute_scheme("DROP TABLE stream_table") + ) + + def _test_stream_cursor_interactive_transaction_guard( + self, connection: dbapi.Connection + ) -> None: + cur = connection.cursor() + maybe_await( + cur.execute_scheme("DROP TABLE IF EXISTS stream_guard_table") + ) + maybe_await( + cur.execute_scheme( + """ + CREATE TABLE stream_guard_table ( + id Int64 NOT NULL, + val Int64, + PRIMARY KEY(id) + ) + """ + ) + ) + maybe_await( + cur.execute( + "UPSERT INTO stream_guard_table (id, val) " + "VALUES (0,0),(1,1),(2,2),(3,3)" + ) + ) + maybe_await(cur.close()) + + connection.set_isolation_level(dbapi.IsolationLevel.SERIALIZABLE) + maybe_await(connection.begin()) + + stream = connection.cursor(stream_results=True) + maybe_await( + stream.execute("SELECT id FROM stream_guard_table ORDER BY id") + ) + assert connection._current_cursor is stream + + other = connection.cursor() + with pytest.raises(dbapi.ProgrammingError): + maybe_await(other.execute("SELECT id FROM stream_guard_table")) + + with pytest.raises(dbapi.ProgrammingError): + maybe_await(connection.commit()) + + rows = [] + while True: + row = maybe_await(stream.fetchone()) + if row is None: + break + rows.append(row) + assert len(rows) == 4 + assert connection._current_cursor is None + + maybe_await(connection.rollback()) + + maybe_await(other.close()) + maybe_await(stream.close()) + maybe_await( + connection.cursor().execute_scheme("DROP TABLE stream_guard_table") + ) + def _test_connection(self, connection: dbapi.Connection) -> None: maybe_await(connection.commit()) maybe_await(connection.rollback()) @@ -208,9 +302,12 @@ def _test_cursor_raw_query(self, connection: dbapi.Connection) -> None: with suppress(dbapi.DatabaseError): maybe_await(cur.execute_scheme("DROP TABLE test")) - maybe_await(cur.execute_scheme( - "CREATE TABLE test(id Int64 NOT NULL, text Utf8, PRIMARY KEY (id))" - )) + maybe_await( + cur.execute_scheme( + "CREATE TABLE test(" + "id Int64 NOT NULL, text Utf8, PRIMARY KEY (id))" + ) + ) maybe_await( cur.execute( @@ -589,9 +686,7 @@ def test_commit_rollback_after_begin( isolation_level: str, connection: dbapi.Connection, ) -> None: - self._test_commit_rollback_after_begin( - connection, isolation_level - ) + self._test_commit_rollback_after_begin(connection, isolation_level) def test_close_releases_session_after_rollback_error( self, connection: dbapi.Connection @@ -632,15 +727,19 @@ def test_errors_with_interactive_tx( ) -> None: self._test_error_with_interactive_tx(connection) - def test_get_view_names( - self, connection: dbapi.Connection - ) -> None: + def test_get_view_names(self, connection: dbapi.Connection) -> None: self._test_get_view_names(connection) - def test_get_table_names( + def test_get_table_names(self, connection: dbapi.Connection) -> None: + self._test_get_table_names(connection) + + def test_stream_cursor_fetch(self, connection: dbapi.Connection) -> None: + self._test_stream_cursor_fetch(connection) + + def test_stream_cursor_interactive_transaction_guard( self, connection: dbapi.Connection ) -> None: - self._test_get_table_names(connection) + self._test_stream_cursor_interactive_transaction_guard(connection) class TestAsyncConnection(BaseDBApiTestSuit): @@ -717,9 +816,7 @@ async def test_commit_rollback_after_begin( connection: dbapi.AsyncConnection, ) -> None: await greenlet_spawn( - self._test_commit_rollback_after_begin, - connection, - isolation_level + self._test_commit_rollback_after_begin, connection, isolation_level ) @pytest.mark.asyncio @@ -796,3 +893,18 @@ async def test_get_table_names( self, connection: dbapi.AsyncConnection ) -> None: await greenlet_spawn(self._test_get_table_names, connection) + + @pytest.mark.asyncio + async def test_stream_cursor_fetch( + self, connection: dbapi.AsyncConnection + ) -> None: + await greenlet_spawn(self._test_stream_cursor_fetch, connection) + + @pytest.mark.asyncio + async def test_stream_cursor_interactive_transaction_guard( + self, connection: dbapi.AsyncConnection + ) -> None: + await greenlet_spawn( + self._test_stream_cursor_interactive_transaction_guard, + connection, + ) diff --git a/tests/test_convert_parameters.py b/tests/test_convert_parameters.py index c0322fe..f6d5fef 100644 --- a/tests/test_convert_parameters.py +++ b/tests/test_convert_parameters.py @@ -109,9 +109,7 @@ def test_multiple_params(self): assert "$c" in p def test_percent_percent_escape(self): - q, _ = convert_query_parameters( - "SELECT %% as pct, %(x)s", {"x": 1} - ) + q, _ = convert_query_parameters("SELECT %% as pct, %(x)s", {"x": 1}) assert q == "SELECT % as pct, $x" def test_empty_params(self): diff --git a/tests/test_cursors.py b/tests/test_cursors.py index b10de80..393e502 100644 --- a/tests/test_cursors.py +++ b/tests/test_cursors.py @@ -9,8 +9,11 @@ import ydb_dbapi from sqlalchemy.util import await_only from sqlalchemy.util import greenlet_spawn +from typing_extensions import Self from ydb_dbapi import AsyncCursor +from ydb_dbapi import AsyncStreamCursor from ydb_dbapi import Cursor +from ydb_dbapi import StreamCursor from ydb_dbapi.utils import CursorStatus @@ -334,3 +337,386 @@ async def test_cursor_state_after_error( self, async_cursor: AsyncCursor ) -> None: await greenlet_spawn(self._test_cursor_state_after_error, async_cursor) + + +class FakeResultSet: + def __init__(self, rows: list[tuple], columns: list[object]) -> None: + self.rows = rows + self.columns = columns + + +def make_stream_result_sets(count: int = 1) -> list[FakeResultSet]: + return [ + FakeResultSet( + rows=[(row_id, row_id) for row_id in range(RESULT_SET_LENGTH)], + columns=[], + ) + for _ in range(count) + ] + + +class FakeSyncResponseContextIterator: + def __init__(self, result_sets: list[FakeResultSet]) -> None: + self._result_sets = iter(result_sets) + self.cancelled = False + + def __iter__(self) -> FakeSyncResponseContextIterator: + return self + + def __next__(self) -> FakeResultSet: + return next(self._result_sets) + + def cancel(self) -> None: + self.cancelled = True + + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type: object, exc: object, tb: object) -> None: + for _ in self: + pass + + +class FakeAsyncResponseContextIterator: + def __init__(self, result_sets: list[FakeResultSet]) -> None: + self._result_sets = iter(result_sets) + self.cancelled = False + + def __aiter__(self) -> FakeAsyncResponseContextIterator: + return self + + async def __anext__(self) -> FakeResultSet: + try: + return next(self._result_sets) + except StopIteration as e: + raise StopAsyncIteration from e + + def cancel(self) -> None: + self.cancelled = True + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, exc_type: object, exc: object, tb: object + ) -> None: + async for _ in self: + pass + + +class FakeFailingSyncResponseContextIterator(FakeSyncResponseContextIterator): + def __next__(self) -> FakeResultSet: + raise ydb.issues.GenericError("boom") + + +class FakeFailingAsyncResponseContextIterator( + FakeAsyncResponseContextIterator +): + async def __anext__(self) -> FakeResultSet: + raise ydb.issues.GenericError("boom") + + +class FakeSyncStreamSession: + def __init__(self, result_sets: list[FakeResultSet]) -> None: + self._result_sets = result_sets + self.deleted = False + + def transaction( + self, tx_mode: ydb.BaseQueryTxMode + ) -> FakeSyncStreamSession: + return self + + def execute(self, **kwargs: object) -> FakeSyncResponseContextIterator: + return FakeSyncResponseContextIterator(self._result_sets) + + def delete(self) -> None: + self.deleted = True + + +class FakeAsyncStreamSession: + def __init__(self, result_sets: list[FakeResultSet]) -> None: + self._result_sets = result_sets + self.deleted = False + + def transaction( + self, tx_mode: ydb.BaseQueryTxMode + ) -> FakeAsyncStreamSession: + return self + + async def execute( + self, **kwargs: object + ) -> FakeAsyncResponseContextIterator: + return FakeAsyncResponseContextIterator(self._result_sets) + + async def delete(self) -> None: + self.deleted = True + + +class FakeFailingSyncStreamSession(FakeSyncStreamSession): + def execute( + self, **kwargs: object + ) -> FakeFailingSyncResponseContextIterator: + return FakeFailingSyncResponseContextIterator(self._result_sets) + + +class FakeFailingAsyncStreamSession(FakeAsyncStreamSession): + async def execute( + self, **kwargs: object + ) -> FakeFailingAsyncResponseContextIterator: + return FakeFailingAsyncResponseContextIterator(self._result_sets) + + +class FakeSyncStreamSessionPool: + def __init__(self, result_sets: list[FakeResultSet]) -> None: + self.session = FakeSyncStreamSession(result_sets) + self.released_sessions: list[FakeSyncStreamSession] = [] + + def acquire(self, timeout: float | None = None) -> FakeSyncStreamSession: + return self.session + + def release(self, session: FakeSyncStreamSession) -> None: + self.released_sessions.append(session) + + +class FakeAsyncStreamSessionPool: + def __init__(self, result_sets: list[FakeResultSet]) -> None: + self.session = FakeAsyncStreamSession(result_sets) + self.released_sessions: list[FakeAsyncStreamSession] = [] + + async def acquire( + self, timeout: float | None = None + ) -> FakeAsyncStreamSession: + return self.session + + async def release(self, session: FakeAsyncStreamSession) -> None: + self.released_sessions.append(session) + + +class FakeSyncStreamConnection: + _tx_context = None + + def __init__(self) -> None: + self.current_cursor: StreamCursor | None = None + + def _invalidate_session(self) -> None: ... + + def _set_current_cursor(self, cursor: StreamCursor) -> None: + self.current_cursor = cursor + + def _clear_current_cursor(self, cursor: Cursor | None = None) -> None: + if cursor is None or self.current_cursor is cursor: + self.current_cursor = None + + +class FakeAsyncStreamConnection: + _tx_context = None + + def __init__(self) -> None: + self.current_cursor: AsyncStreamCursor | None = None + + async def _invalidate_session(self) -> None: ... + + def _set_current_cursor(self, cursor: AsyncStreamCursor) -> None: + self.current_cursor = cursor + + def _clear_current_cursor(self, cursor: AsyncCursor | None = None) -> None: + if cursor is None or self.current_cursor is cursor: + self.current_cursor = None + + +class TestStreamCursor: + def _make_cursor(self, result_sets: list) -> StreamCursor: + return StreamCursor( + FakeSyncStreamConnection(), + FakeSyncStreamSessionPool(result_sets), + ydb.QuerySerializableReadWrite(), + request_settings=ydb.BaseRequestSettings(), + retry_settings=ydb.RetrySettings(), + ) + + def test_stream_cursor_fetch_one(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + try: + cursor.execute("SELECT id, val FROM table") + + assert cursor.rowcount == -1 + + for i in range(RESULT_SET_LENGTH): + row = cursor.fetchone() + assert row is not None + assert row[0] == i + + assert cursor.fetchone() is None + assert cursor.rowcount == RESULT_SET_LENGTH + finally: + cursor.close() + + def test_stream_cursor_fetch_many(self) -> None: + cursor = self._make_cursor(make_stream_result_sets(RESULT_SET_COUNT)) + try: + cursor.execute( + """ + SELECT id, val FROM table; + SELECT id, val FROM table1; + SELECT id, val FROM table2; + """ + ) + + assert cursor.rowcount == -1 + + rows = cursor.fetchmany(size=5) + assert len(rows) == 5 + assert cursor.rowcount == -1 + + rows = cursor.fetchmany(size=7) + assert len(rows) == 7 + assert cursor.fetchmany(size=1) == [] + assert cursor.rowcount == RESULT_SET_LENGTH * RESULT_SET_COUNT + finally: + cursor.close() + + def test_stream_cursor_fetch_all(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + try: + cursor.execute("SELECT id, val FROM table") + + rows = cursor.fetchall() + assert len(rows) == RESULT_SET_LENGTH + assert cursor.rowcount == RESULT_SET_LENGTH + assert cursor.fetchall() == [] + finally: + cursor.close() + + def test_stream_cursor_releases_session_after_drain(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + pool = cursor._session_pool + try: + cursor.execute("SELECT id, val FROM table") + cursor.fetchall() + + assert pool.released_sessions == [pool.session] + assert pool.session.deleted is False + finally: + cursor.close() + + def test_stream_cursor_discards_session_on_close(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + pool = cursor._session_pool + cursor.execute("SELECT id, val FROM table") + cursor.close() + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + + def test_stream_cursor_cancels_stream_on_close(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + cursor.execute("SELECT id, val FROM table") + stream = cursor._stream + cursor.close() + + assert stream is not None + assert stream.cancelled is True + + def test_stream_cursor_invalidate_owner_session_on_error(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + pool = cursor._session_pool + pool.session = FakeFailingSyncStreamSession(make_stream_result_sets()) + + with pytest.raises(ydb_dbapi.Error): + cursor.execute("SELECT id, val FROM table") + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + assert cursor._state == CursorStatus.finished + + +class TestAsyncStreamCursor: + def _make_cursor(self, result_sets: list) -> AsyncStreamCursor: + return AsyncStreamCursor( + FakeAsyncStreamConnection(), + FakeAsyncStreamSessionPool(result_sets), + ydb.QuerySerializableReadWrite(), + request_settings=ydb.BaseRequestSettings(), + retry_settings=ydb.RetrySettings(), + ) + + @pytest.mark.asyncio + async def test_stream_cursor_fetch_one(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + try: + await cursor.execute("SELECT id, val FROM table") + + assert cursor.rowcount == -1 + + for i in range(RESULT_SET_LENGTH): + row = await cursor.fetchone() + assert row is not None + assert row[0] == i + + assert await cursor.fetchone() is None + assert cursor.rowcount == RESULT_SET_LENGTH + finally: + await cursor.close() + + @pytest.mark.asyncio + async def test_stream_cursor_fetch_many(self) -> None: + cursor = self._make_cursor(make_stream_result_sets(RESULT_SET_COUNT)) + try: + await cursor.execute( + """ + SELECT id, val FROM table; + SELECT id, val FROM table1; + SELECT id, val FROM table2; + """ + ) + + assert cursor.rowcount == -1 + + rows = await cursor.fetchmany(size=5) + assert len(rows) == 5 + assert cursor.rowcount == -1 + + rows = await cursor.fetchmany(size=7) + assert len(rows) == 7 + assert await cursor.fetchmany(size=1) == [] + assert cursor.rowcount == RESULT_SET_LENGTH * RESULT_SET_COUNT + finally: + await cursor.close() + + @pytest.mark.asyncio + async def test_stream_cursor_fetch_all(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + try: + await cursor.execute("SELECT id, val FROM table") + + rows = await cursor.fetchall() + assert len(rows) == RESULT_SET_LENGTH + assert cursor.rowcount == RESULT_SET_LENGTH + assert await cursor.fetchall() == [] + finally: + await cursor.close() + + @pytest.mark.asyncio + async def test_stream_cursor_discards_session_on_close(self) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + pool = cursor._session_pool + await cursor.execute("SELECT id, val FROM table") + await cursor.close() + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + + @pytest.mark.asyncio + async def test_stream_cursor_invalidate_owner_session_on_error( + self, + ) -> None: + cursor = self._make_cursor(make_stream_result_sets()) + pool = cursor._session_pool + pool.session = FakeFailingAsyncStreamSession(make_stream_result_sets()) + + with pytest.raises(ydb_dbapi.Error): + await cursor.execute("SELECT id, val FROM table") + + assert pool.session.deleted is True + assert pool.released_sessions == [pool.session] + assert cursor._state == CursorStatus.finished diff --git a/tests/test_stream_memory.py b/tests/test_stream_memory.py new file mode 100644 index 0000000..58dd861 --- /dev/null +++ b/tests/test_stream_memory.py @@ -0,0 +1,157 @@ +import tracemalloc +from collections.abc import AsyncGenerator +from collections.abc import Generator +from inspect import iscoroutine + +import pytest +import pytest_asyncio +import ydb_dbapi as dbapi +from sqlalchemy.util import await_only +from sqlalchemy.util import greenlet_spawn + +ROW_COUNT = 100_000 +BATCH_SIZE = 1_000 +PAYLOAD_SIZE = 256 +TABLE_NAME = "stream_memory_table" + +ConnectionType = dbapi.Connection | dbapi.AsyncConnection + + +def maybe_await(obj: callable) -> any: + if not iscoroutine(obj): + return obj + return await_only(obj) + + +def _create_table(connection: ConnectionType) -> None: + cursor = connection.cursor() + try: + maybe_await( + cursor.execute_scheme(f"DROP TABLE IF EXISTS {TABLE_NAME}") + ) + maybe_await( + cursor.execute_scheme( + f""" + CREATE TABLE {TABLE_NAME} ( + id Int64 NOT NULL, + payload Utf8, + PRIMARY KEY(id) + ) + """ + ) + ) + payload = "x" * PAYLOAD_SIZE + for start in range(0, ROW_COUNT, BATCH_SIZE): + stop = min(start + BATCH_SIZE, ROW_COUNT) + values = ",".join( + f"({row_id}, '{payload}')" for row_id in range(start, stop) + ) + maybe_await( + cursor.execute( + f"UPSERT INTO {TABLE_NAME} (id, payload) " + f"VALUES {values}" + ) + ) + finally: + maybe_await(cursor.close()) + + +def _drop_table(connection: ConnectionType) -> None: + maybe_await( + connection.cursor().execute_scheme( + f"DROP TABLE IF EXISTS {TABLE_NAME}" + ) + ) + + +def _measure_buffered(connection: ConnectionType) -> tuple[int, int]: + tracemalloc.start() + try: + cursor = connection.cursor() + maybe_await(cursor.execute(f"SELECT id, payload FROM {TABLE_NAME}")) + rows = maybe_await(cursor.fetchall()) + row_count = len(rows) + del rows + maybe_await(cursor.close()) + _, peak = tracemalloc.get_traced_memory() + return row_count, peak + finally: + tracemalloc.stop() + + +def _measure_stream(connection: ConnectionType) -> tuple[int, int]: + tracemalloc.start() + try: + cursor = connection.cursor(stream_results=True) + maybe_await(cursor.execute(f"SELECT id, payload FROM {TABLE_NAME}")) + row_count = 0 + while maybe_await(cursor.fetchone()) is not None: + row_count += 1 + assert cursor.rowcount == ROW_COUNT + maybe_await(cursor.close()) + _, peak = tracemalloc.get_traced_memory() + return row_count, peak + finally: + tracemalloc.stop() + + +def _run_stream_memory_test(connection: ConnectionType) -> None: + _create_table(connection) + try: + buffered_rows, buffered_peak = _measure_buffered(connection) + stream_rows, stream_peak = _measure_stream(connection) + + print( # noqa: T201 + f"\nBUFFERED peak: {buffered_peak / 1024 / 1024:.1f} MiB " + f"({buffered_rows} rows) | STREAM peak: " + f"{stream_peak / 1024 / 1024:.1f} MiB ({stream_rows} rows)" + ) + + assert buffered_rows == ROW_COUNT + assert stream_rows == ROW_COUNT + assert buffered_peak > 10 * 1024 * 1024 + assert stream_peak * 3 < buffered_peak + finally: + _drop_table(connection) + + +@pytest.fixture +def connection( + connection_kwargs: dict, +) -> Generator[dbapi.Connection]: + conn = dbapi.connect(**connection_kwargs) + try: + yield conn + finally: + conn.close() + + +@pytest_asyncio.fixture +async def async_connection( + connection_kwargs: dict, +) -> AsyncGenerator[dbapi.AsyncConnection]: + def connect() -> dbapi.AsyncConnection: + return maybe_await(dbapi.async_connect(**connection_kwargs)) + + conn = await greenlet_spawn(connect) + try: + yield conn + finally: + + def close() -> None: + maybe_await(conn.close()) + + await greenlet_spawn(close) + + +def test_stream_cursor_keeps_memory_bounded( + connection: dbapi.Connection, +) -> None: + _run_stream_memory_test(connection) + + +@pytest.mark.asyncio +async def test_async_stream_cursor_keeps_memory_bounded( + async_connection: dbapi.AsyncConnection, +) -> None: + await greenlet_spawn(_run_stream_memory_test, async_connection) diff --git a/ydb_dbapi/__init__.py b/ydb_dbapi/__init__.py index 1d2f2dd..bd79565 100644 --- a/ydb_dbapi/__init__.py +++ b/ydb_dbapi/__init__.py @@ -5,7 +5,9 @@ from .connections import connect from .constants import * from .cursors import AsyncCursor +from .cursors import AsyncStreamCursor from .cursors import Cursor +from .cursors import StreamCursor from .errors import * from .version import VERSION diff --git a/ydb_dbapi/connections.py b/ydb_dbapi/connections.py index 827b5bb..984dc8b 100644 --- a/ydb_dbapi/connections.py +++ b/ydb_dbapi/connections.py @@ -15,11 +15,14 @@ from ydb.retries import retry_operation_sync from .cursors import AsyncCursor +from .cursors import AsyncStreamCursor from .cursors import Cursor +from .cursors import StreamCursor from .errors import InterfaceError from .errors import InternalError from .errors import NotSupportedError from .errors import ProgrammingError +from .utils import CursorStatus from .utils import handle_ydb_errors from .utils import maybe_get_current_trace_id from .utils import prepare_credentials @@ -228,6 +231,7 @@ class Connection(BaseConnection): _driver_cls = ydb.Driver _pool_cls = ydb.QuerySessionPool _cursor_cls = Cursor + _stream_cursor_cls = StreamCursor def __init__( self, @@ -258,10 +262,13 @@ def __init__( pyformat=pyformat, **kwargs, ) - self._current_cursor: Cursor | None = None + self._current_cursor: Cursor | StreamCursor | None = None - def cursor(self) -> Cursor: - return self._cursor_cls( + def cursor(self, stream_results: bool = False) -> Cursor | StreamCursor: + cursor_cls = ( + self._stream_cursor_cls if stream_results else self._cursor_cls + ) + return cursor_cls( connection=self, session_pool=self._session_pool, tx_mode=self._tx_mode, @@ -291,8 +298,15 @@ def begin(self) -> None: self._session = self._session_pool.acquire() self._tx_context = self._session.transaction(self._tx_mode) - @handle_ydb_errors def commit(self) -> None: + self._raise_if_current_cursor_running() + try: + self._commit() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + def _commit(self) -> None: if self._tx_context: settings = self._get_request_settings() self._tx_context.commit(settings=settings) @@ -301,8 +315,15 @@ def commit(self) -> None: self._session_pool.release(self._session) self._session = None - @handle_ydb_errors def rollback(self) -> None: + self._raise_if_current_cursor_running() + try: + self._rollback() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + def _rollback(self) -> None: if self._tx_context: settings = self._get_request_settings() self._tx_context.rollback(settings=settings) @@ -313,6 +334,8 @@ def rollback(self) -> None: @handle_ydb_errors def close(self) -> None: + if self._current_cursor is not None: + self._current_cursor.close() try: self.rollback() finally: @@ -418,17 +441,38 @@ def bulk_upsert( ) def _invalidate_session(self) -> None: + self._clear_current_cursor() if self._tx_context: self._tx_context = None if self._session: self._session_pool.release(self._session) self._session = None + def _set_current_cursor(self, cursor: Cursor | StreamCursor) -> None: + self._current_cursor = cursor + + def _clear_current_cursor(self, cursor: Cursor | None = None) -> None: + if cursor is None or self._current_cursor is cursor: + self._current_cursor = None + + def _raise_if_current_cursor_running( + self, cursor: Cursor | None = None + ) -> None: + if self._current_cursor is None or self._current_cursor is cursor: + return + if self._current_cursor._state != CursorStatus.running: + return + raise ProgrammingError( + "Could not perform operation: a server-side cursor is still " + "streaming results for the current transaction session." + ) + class AsyncConnection(BaseConnection): _driver_cls = ydb.aio.Driver _pool_cls = ydb.aio.QuerySessionPool _cursor_cls = AsyncCursor + _stream_cursor_cls = AsyncStreamCursor def __init__( self, @@ -459,10 +503,15 @@ def __init__( pyformat=pyformat, **kwargs, ) - self._current_cursor: AsyncCursor | None = None + self._current_cursor: AsyncStreamCursor | None = None - def cursor(self) -> AsyncCursor: - return self._cursor_cls( + def cursor( + self, stream_results: bool = False + ) -> AsyncCursor | AsyncStreamCursor: + cursor_cls = ( + self._stream_cursor_cls if stream_results else self._cursor_cls + ) + return cursor_cls( connection=self, session_pool=self._session_pool, tx_mode=self._tx_mode, @@ -492,8 +541,15 @@ async def begin(self) -> None: self._session = await self._session_pool.acquire() self._tx_context = self._session.transaction(self._tx_mode) - @handle_ydb_errors async def commit(self) -> None: + self._raise_if_current_cursor_running() + try: + await self._commit() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + async def _commit(self) -> None: if self._tx_context: settings = self._get_request_settings() await self._tx_context.commit(settings=settings) @@ -502,8 +558,15 @@ async def commit(self) -> None: await self._session_pool.release(self._session) self._session = None - @handle_ydb_errors async def rollback(self) -> None: + self._raise_if_current_cursor_running() + try: + await self._rollback() + finally: + self._clear_current_cursor() + + @handle_ydb_errors + async def _rollback(self) -> None: if self._tx_context: settings = self._get_request_settings() await self._tx_context.rollback(settings=settings) @@ -514,6 +577,8 @@ async def rollback(self) -> None: @handle_ydb_errors async def close(self) -> None: + if self._current_cursor is not None: + await self._current_cursor.close() try: await self.rollback() finally: @@ -623,12 +688,34 @@ async def bulk_upsert( ) async def _invalidate_session(self) -> None: + self._clear_current_cursor() if self._tx_context: self._tx_context = None if self._session: await self._session_pool.release(self._session) self._session = None + def _set_current_cursor(self, cursor: AsyncStreamCursor) -> None: + self._current_cursor = cursor + + def _clear_current_cursor( + self, cursor: AsyncStreamCursor | None = None + ) -> None: + if cursor is None or self._current_cursor is cursor: + self._current_cursor = None + + def _raise_if_current_cursor_running( + self, cursor: AsyncStreamCursor | None = None + ) -> None: + if self._current_cursor is None or self._current_cursor is cursor: + return + if self._current_cursor._state != CursorStatus.running: + return + raise ProgrammingError( + "Could not perform operation: a server-side cursor is still " + "streaming results for the current transaction session." + ) + def connect( host: str = "", diff --git a/ydb_dbapi/cursors.py b/ydb_dbapi/cursors.py index daa0955..e79f4c7 100644 --- a/ydb_dbapi/cursors.py +++ b/ydb_dbapi/cursors.py @@ -13,6 +13,10 @@ from typing import Union import ydb +from ydb.aio.query.base import AsyncResponseContextIterator +from ydb.query.base import SyncResponseContextIterator +from ydb.retries import retry_operation_async +from ydb.retries import retry_operation_sync from .errors import DatabaseError from .errors import InterfaceError @@ -57,7 +61,7 @@ async def awrapper( return await func(self, *args, **kwargs) except ydb.Error: self._state = CursorStatus.finished - await self._connection._invalidate_session() + await self._invalidate_active_session() raise return awrapper @@ -68,7 +72,7 @@ def wrapper(self: Cursor, *args: tuple, **kwargs: dict) -> Any: return func(self, *args, **kwargs) except ydb.Error: self._state = CursorStatus.finished - self._connection._invalidate_session() + self._invalidate_active_session() raise return wrapper @@ -249,6 +253,9 @@ def _get_request_settings(self) -> ydb.BaseRequestSettings: return settings + def _invalidate_active_session(self) -> None: + self._connection._invalidate_session() + def _materialize( self, stream: Iterator[ydb.convert.ResultSet] ) -> list[ydb.convert.ResultSet]: @@ -327,6 +334,7 @@ def execute_scheme( parameters: ParametersType | None = None, ) -> None: self._raise_if_closed() + self._raise_if_running() query = self._append_table_path_prefix(query) self._begin_query() @@ -344,6 +352,8 @@ def execute( ) -> None: self._raise_if_closed() self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) @@ -391,6 +401,265 @@ def __exit__( self.close() +class StreamCursor(Cursor): + def __init__( + self, + connection: Connection, + session_pool: ydb.QuerySessionPool, + tx_mode: ydb.BaseQueryTxMode, + request_settings: ydb.BaseRequestSettings, + retry_settings: ydb.RetrySettings, + table_path_prefix: str = "", + pyformat: bool = False, + ) -> None: + super().__init__( + connection=connection, + session_pool=session_pool, + tx_mode=tx_mode, + request_settings=request_settings, + retry_settings=retry_settings, + table_path_prefix=table_path_prefix, + pyformat=pyformat, + ) + self._stream: SyncResponseContextIterator | None = None + self._session_owner: ydb.QuerySession | None = None + self._rowcount_accumulator: int = 0 + + def _invalidate_active_session(self) -> None: + self._clear_current_cursor() + if self._session_owner is None: + self._connection._invalidate_session() + return + + session = self._session_owner + self._session_owner = None + self._stream = None + try: + session.delete() + finally: + self._session_pool.release(session) + + def _clear_current_cursor(self) -> None: + self._connection._clear_current_cursor(self) + + def _register_current_cursor(self) -> None: + if self._tx_context is not None: + self._connection._set_current_cursor(self) + + def _release_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + self._session_pool.release(session) + + def _discard_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + try: + session.delete() + finally: + self._session_pool.release(session) + + def _finish_stream(self) -> None: + self._stream = None + self._release_owned_session() + self._clear_current_cursor() + self._finish_query() + + def _reset_result(self) -> None: + super()._reset_result() + self._rowcount_accumulator = 0 + + def _update_result_set( + self, + result_set: ydb.convert.ResultSet, + replace_current: bool = True, + ) -> None: + # A streaming cursor reports the total number of fetched rows only + # after the stream is exhausted, so here the count is accumulated + # and the final value is set in _finish_query(). + self._update_description(result_set) + self._rowcount_accumulator += len(result_set.rows) + + new_rows_iter = self._rows_iterable(result_set) + + if self._rows is None or replace_current: + self._rows = new_rows_iter + else: + self._rows = itertools.chain(self._rows, new_rows_iter) + + def _finalize_rowcount(self) -> None: + self._rows_count = self._rowcount_accumulator + + def _finish_query(self) -> None: + self._finalize_rowcount() + super()._finish_query() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + def _load_next_result_set(self) -> bool: + if self._stream is None: + return False + + try: + result_set = next(self._stream) + except StopIteration: + self._finish_stream() + return False + + self._update_result_set(result_set, replace_current=False) + return True + + def _prime_stream(self) -> None: + if ( + not self._load_next_result_set() + and self._state == CursorStatus.running + ): + self._finish_stream() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + def _execute_session_query_stream( + self, + query: str, + parameters: ParametersType | None = None, + ) -> SyncResponseContextIterator: + settings = self._get_request_settings() + + def callee() -> SyncResponseContextIterator: + acquire_timeout = getattr( + self._retry_settings, + "max_session_acquire_timeout", + None, + ) + session = self._session_pool.acquire(timeout=acquire_timeout) + try: + stream = session.transaction(self._tx_mode).execute( + query=query, + parameters=parameters, + commit_tx=True, + settings=settings, + ) + except Exception: + self._session_pool.release(session) + raise + + self._session_owner = session + return stream + + return retry_operation_sync( + callee, + self._retry_settings, + ) + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + def _execute_transactional_query_stream( + self, + tx_context: ydb.QueryTxContext, + query: str, + parameters: ParametersType | None = None, + ) -> SyncResponseContextIterator: + settings = self._get_request_settings() + return tx_context.execute( + query=query, + parameters=parameters, + commit_tx=False, + settings=settings, + ) + + def execute( + self, + query: str, + parameters: ParametersType | None = None, + ) -> None: + self._raise_if_closed() + self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running(self) + + query = self._append_table_path_prefix(query) + + if self._pyformat and parameters is not None: + query, parameters = convert_query_parameters(query, parameters) + + self._begin_query() + + if self._tx_context is not None: + self._stream = self._execute_transactional_query_stream( + tx_context=self._tx_context, + query=query, + parameters=parameters, + ) + self._register_current_cursor() + else: + self._stream = self._execute_session_query_stream( + query=query, + parameters=parameters, + ) + + self._prime_stream() + + def fetchone(self) -> tuple | None: + self._raise_if_closed() + + while True: + row = self._fetchone_from_buffer() + if row is not None: + return row + if not self._load_next_result_set(): + return None + + def fetchmany(self, size: int | None = None) -> list: + self._raise_if_closed() + result: list[tuple] = [] + target_size = size or self.arraysize + + while len(result) < target_size: + row = self.fetchone() + if row is None: + break + result.append(row) + + return result + + def fetchall(self) -> list: + self._raise_if_closed() + result = list(self._fetchall_from_buffer()) + + while self._load_next_result_set(): + result.extend(self._fetchall_from_buffer()) + + return result + + def close(self) -> None: + if self._state == CursorStatus.closed: + return + + if self._state == CursorStatus.running: + if self._session_owner is not None and self._stream is not None: + self._stream.cancel() + self._stream = None + self._discard_owned_session() + elif self._stream is not None: + with self._stream: + pass + self._finish_stream() + else: + self._release_owned_session() + self._clear_current_cursor() + + self._stream = None + self._session_owner = None + self._clear_current_cursor() + self._state = CursorStatus.closed + + class AsyncCursor(BufferedCursor): def __init__( self, @@ -436,6 +705,9 @@ def _get_request_settings(self) -> ydb.BaseRequestSettings: return settings + async def _invalidate_active_session(self) -> None: + await self._connection._invalidate_session() + async def _materialize( self, stream: AsyncIterator[ydb.convert.ResultSet] ) -> list[ydb.convert.ResultSet]: @@ -514,6 +786,7 @@ async def execute_scheme( parameters: ParametersType | None = None, ) -> None: self._raise_if_closed() + self._raise_if_running() query = self._append_table_path_prefix(query) self._begin_query() @@ -531,6 +804,8 @@ async def execute( ) -> None: self._raise_if_closed() self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) @@ -576,3 +851,340 @@ async def __aexit__( tb: object, ) -> None: self.close() + + +class AsyncStreamCursor(BufferedCursor): + def __init__( + self, + connection: AsyncConnection, + session_pool: ydb.aio.QuerySessionPool, + tx_mode: ydb.BaseQueryTxMode, + request_settings: ydb.BaseRequestSettings, + retry_settings: ydb.RetrySettings, + table_path_prefix: str = "", + pyformat: bool = False, + ) -> None: + super().__init__() + self._connection = connection + self._session_pool = session_pool + self._tx_mode = tx_mode + self._request_settings = request_settings + self._retry_settings = retry_settings + self._table_path_prefix = table_path_prefix + self._pyformat = pyformat + self._stream: AsyncResponseContextIterator | None = None + self._session_owner: ydb.aio.QuerySession | None = None + self._rowcount_accumulator: int = 0 + + @property + def _tx_context(self) -> ydb.aio.QueryTxContext | None: + # Read the transaction from the connection on every access: a cursor + # must not pin the transaction that was open when it was created. + return self._connection._tx_context + + def _get_request_settings(self) -> ydb.BaseRequestSettings: + settings = self._request_settings.make_copy() + + if self._request_settings.trace_id is None: + settings = settings.with_trace_id(maybe_get_current_trace_id()) + + return settings + + async def _materialize( + self, stream: AsyncIterator[ydb.convert.ResultSet] + ) -> list[ydb.convert.ResultSet]: + return [result_set async for result_set in stream] + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _execute_generic_query( + self, query: str, parameters: ParametersType | None = None + ) -> list[ydb.convert.ResultSet]: + settings = self._get_request_settings() + + async def callee( + session: ydb.aio.QuerySession, + ) -> list[ydb.convert.ResultSet]: + return await self._materialize( + await session.execute( + query=query, + parameters=parameters, + settings=settings, + ) + ) + + return await self._session_pool.retry_operation_async( + callee, + retry_settings=self._retry_settings, + ) + + async def execute_scheme( + self, + query: str, + parameters: ParametersType | None = None, + ) -> None: + self._raise_if_closed() + self._raise_if_running() + + query = self._append_table_path_prefix(query) + self._begin_query() + + result_list = await self._execute_generic_query( + query=query, parameters=parameters + ) + self._fill_buffer(result_list) + self._finish_query() + + async def executemany( + self, query: str, seq_of_parameters: Sequence[ParametersType] + ) -> None: + for parameters in seq_of_parameters: + await self.execute(query, parameters) + + async def _invalidate_active_session(self) -> None: + self._clear_current_cursor() + self._stream = None + if self._session_owner is None: + await self._connection._invalidate_session() + return + + session = self._session_owner + self._session_owner = None + try: + await session.delete() + finally: + await self._session_pool.release(session) + + def _clear_current_cursor(self) -> None: + self._connection._clear_current_cursor(self) + + def _register_current_cursor(self) -> None: + if self._tx_context is not None: + self._connection._set_current_cursor(self) + + async def _release_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + await self._session_pool.release(session) + + async def _discard_owned_session(self) -> None: + if self._session_owner is None: + return + + session = self._session_owner + self._session_owner = None + try: + await session.delete() + finally: + await self._session_pool.release(session) + + async def _finish_stream(self) -> None: + self._stream = None + await self._release_owned_session() + self._clear_current_cursor() + self._finish_query() + + def _reset_result(self) -> None: + super()._reset_result() + self._rowcount_accumulator = 0 + + def _update_result_set( + self, + result_set: ydb.convert.ResultSet, + replace_current: bool = True, + ) -> None: + # A streaming cursor reports the total number of fetched rows only + # after the stream is exhausted, so here the count is accumulated + # and the final value is set in _finish_query(). + self._update_description(result_set) + self._rowcount_accumulator += len(result_set.rows) + + new_rows_iter = self._rows_iterable(result_set) + + if self._rows is None or replace_current: + self._rows = new_rows_iter + else: + self._rows = itertools.chain(self._rows, new_rows_iter) + + def _finalize_rowcount(self) -> None: + self._rows_count = self._rowcount_accumulator + + def _finish_query(self) -> None: + self._finalize_rowcount() + super()._finish_query() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _load_next_result_set(self) -> bool: + if self._stream is None: + return False + + try: + result_set = await self._stream.__anext__() + except StopAsyncIteration: + await self._finish_stream() + return False + + self._update_result_set(result_set, replace_current=False) + return True + + async def _prime_stream(self) -> None: + if ( + not await self._load_next_result_set() + and self._state == CursorStatus.running + ): + await self._finish_stream() + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _execute_session_query_stream( + self, + query: str, + parameters: ParametersType | None = None, + ) -> AsyncResponseContextIterator: + settings = self._get_request_settings() + + async def callee() -> AsyncResponseContextIterator: + acquire_timeout = getattr( + self._retry_settings, + "max_session_acquire_timeout", + None, + ) + session = await self._session_pool.acquire(timeout=acquire_timeout) + try: + stream = await session.transaction(self._tx_mode).execute( + query=query, + parameters=parameters, + commit_tx=True, + settings=settings, + ) + except Exception: + await self._session_pool.release(session) + raise + + self._session_owner = session + return stream + + return await retry_operation_async( + callee, + self._retry_settings, + ) + + @handle_ydb_errors + @invalidate_cursor_on_ydb_error + async def _execute_transactional_query_stream( + self, + tx_context: ydb.aio.QueryTxContext, + query: str, + parameters: ParametersType | None = None, + ) -> AsyncResponseContextIterator: + settings = self._get_request_settings() + return await tx_context.execute( + query=query, + parameters=parameters, + commit_tx=False, + settings=settings, + ) + + async def execute( + self, + query: str, + parameters: ParametersType | None = None, + ) -> None: + self._raise_if_closed() + self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running(self) + + query = self._append_table_path_prefix(query) + + if self._pyformat and parameters is not None: + query, parameters = convert_query_parameters(query, parameters) + + self._begin_query() + + if self._tx_context is not None: + self._stream = await self._execute_transactional_query_stream( + tx_context=self._tx_context, + query=query, + parameters=parameters, + ) + self._register_current_cursor() + else: + self._stream = await self._execute_session_query_stream( + query=query, + parameters=parameters, + ) + + await self._prime_stream() + + async def fetchone(self) -> tuple | None: + self._raise_if_closed() + + while True: + row = self._fetchone_from_buffer() + if row is not None: + return row + if not await self._load_next_result_set(): + return None + + async def fetchmany(self, size: int | None = None) -> list: + self._raise_if_closed() + result: list[tuple] = [] + target_size = size or self.arraysize + + while len(result) < target_size: + row = await self.fetchone() + if row is None: + break + result.append(row) + + return result + + async def fetchall(self) -> list: + self._raise_if_closed() + result = list(self._fetchall_from_buffer()) + + while await self._load_next_result_set(): + result.extend(self._fetchall_from_buffer()) + + return result + + async def nextset(self) -> bool: + return False + + async def close(self) -> None: + if self._state == CursorStatus.closed: + return + + if self._state == CursorStatus.running: + if self._session_owner is not None and self._stream is not None: + self._stream.cancel() + self._stream = None + await self._discard_owned_session() + elif self._stream is not None: + async with self._stream: + pass + await self._finish_stream() + else: + await self._release_owned_session() + self._clear_current_cursor() + + self._stream = None + self._session_owner = None + self._clear_current_cursor() + self._state = CursorStatus.closed + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: object, + ) -> None: + await self.close() diff --git a/ydb_dbapi/utils.py b/ydb_dbapi/utils.py index 1c456e7..1a18a40 100644 --- a/ydb_dbapi/utils.py +++ b/ydb_dbapi/utils.py @@ -409,14 +409,14 @@ def _validate_named_parameters( ) -> dict: if not isinstance(parameters, dict): raise ProgrammingError( - "Named placeholders (%(name)s) require parameters " - "to be a dict." + "Named placeholders (%(name)s) require parameters to be a dict." ) expected_keys = set(named_placeholders) actual_keys = set(parameters) invalid_prefixed_keys = [ - name for name in actual_keys + name + for name in actual_keys if isinstance(name, str) and name.startswith("$") ] if invalid_prefixed_keys: diff --git a/ydb_dbapi/version.py b/ydb_dbapi/version.py index 521bb29..fc0789e 100644 --- a/ydb_dbapi/version.py +++ b/ydb_dbapi/version.py @@ -1 +1 @@ -VERSION = "0.1.23" +VERSION = "0.1.24" From 47ef69a5479f7548be3a0e0bc45ef47663d8c79a Mon Sep 17 00:00:00 2001 From: GrigorySmirnovSpb Date: Tue, 27 Oct 2026 19:19:24 +0300 Subject: [PATCH 2/4] Fix CI error and restored poetry.lock to its initial state --- poetry.lock | 7 ++----- tests/test_stream_memory.py | 5 ++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index d79ae5b..ad5a426 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -3016,10 +3016,7 @@ files = [ [package.dependencies] distlib = ">=0.3.7,<1" -filelock = [ - {version = ">=3.16.1,<=3.19.1", markers = "python_version < \"3.10\""}, - {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""}, -] +filelock = {version = ">=3.16.1,<=3.19.1", markers = "python_version < \"3.10\""} platformdirs = ">=3.9.1,<5" python-discovery = ">=1.4.2" typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} diff --git a/tests/test_stream_memory.py b/tests/test_stream_memory.py index 58dd861..c1f6bed 100644 --- a/tests/test_stream_memory.py +++ b/tests/test_stream_memory.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import tracemalloc from collections.abc import AsyncGenerator from collections.abc import Generator from inspect import iscoroutine +from typing import Union import pytest import pytest_asyncio @@ -14,7 +17,7 @@ PAYLOAD_SIZE = 256 TABLE_NAME = "stream_memory_table" -ConnectionType = dbapi.Connection | dbapi.AsyncConnection +ConnectionType = Union[dbapi.Connection, dbapi.AsyncConnection] def maybe_await(obj: callable) -> any: From 519ad102414f39070468e1e15505830af202f38e Mon Sep 17 00:00:00 2001 From: GrigorySmirnovSpb Date: Tue, 27 Oct 2026 21:18:02 +0300 Subject: [PATCH 3/4] Fixed guards and some mistakes, and added mocks in test --- tests/test_cursors.py | 313 ++++++++++++++++++--------------------- ydb_dbapi/connections.py | 2 + ydb_dbapi/cursors.py | 15 +- 3 files changed, 163 insertions(+), 167 deletions(-) diff --git a/tests/test_cursors.py b/tests/test_cursors.py index 393e502..611eee4 100644 --- a/tests/test_cursors.py +++ b/tests/test_cursors.py @@ -3,6 +3,8 @@ from collections.abc import AsyncGenerator from collections.abc import Generator from inspect import iscoroutine +from typing import Any +from unittest import mock import pytest import ydb @@ -339,31 +341,32 @@ async def test_cursor_state_after_error( await greenlet_spawn(self._test_cursor_state_after_error, async_cursor) -class FakeResultSet: - def __init__(self, rows: list[tuple], columns: list[object]) -> None: - self.rows = rows - self.columns = columns +# Mock-based stream cursor tests -def make_stream_result_sets(count: int = 1) -> list[FakeResultSet]: - return [ - FakeResultSet( - rows=[(row_id, row_id) for row_id in range(RESULT_SET_LENGTH)], - columns=[], - ) - for _ in range(count) - ] +RESULT_SET_LENGTH = 4 +RESULT_SET_COUNT = 3 + +def make_result_sets(count: int = 1) -> list[mock.MagicMock]: + result_sets = [] + for _ in range(count): + rs = mock.MagicMock() + rs.rows = [(row_id, row_id) for row_id in range(RESULT_SET_LENGTH)] + rs.columns = [] + result_sets.append(rs) + return result_sets -class FakeSyncResponseContextIterator: - def __init__(self, result_sets: list[FakeResultSet]) -> None: + +class _SyncStream: + def __init__(self, result_sets: list) -> None: self._result_sets = iter(result_sets) self.cancelled = False - def __iter__(self) -> FakeSyncResponseContextIterator: + def __iter__(self) -> _SyncStream: return self - def __next__(self) -> FakeResultSet: + def __next__(self) -> Any: return next(self._result_sets) def cancel(self) -> None: @@ -372,24 +375,24 @@ def cancel(self) -> None: def __enter__(self) -> Self: return self - def __exit__(self, exc_type: object, exc: object, tb: object) -> None: + def __exit__(self, *args: object) -> None: for _ in self: pass -class FakeAsyncResponseContextIterator: - def __init__(self, result_sets: list[FakeResultSet]) -> None: +class _AsyncStream: + def __init__(self, result_sets: list) -> None: self._result_sets = iter(result_sets) self.cancelled = False - def __aiter__(self) -> FakeAsyncResponseContextIterator: + def __aiter__(self) -> _AsyncStream: return self - async def __anext__(self) -> FakeResultSet: + async def __anext__(self) -> Any: try: return next(self._result_sets) - except StopIteration as e: - raise StopAsyncIteration from e + except StopIteration as exc: + raise StopAsyncIteration from exc def cancel(self) -> None: self.cancelled = True @@ -397,145 +400,105 @@ def cancel(self) -> None: async def __aenter__(self) -> Self: return self - async def __aexit__( - self, exc_type: object, exc: object, tb: object - ) -> None: + async def __aexit__(self, *args: object) -> None: async for _ in self: pass -class FakeFailingSyncResponseContextIterator(FakeSyncResponseContextIterator): - def __next__(self) -> FakeResultSet: +class _FailingSyncStream(_SyncStream): + def __next__(self) -> Any: raise ydb.issues.GenericError("boom") -class FakeFailingAsyncResponseContextIterator( - FakeAsyncResponseContextIterator -): - async def __anext__(self) -> FakeResultSet: +class _FailingAsyncStream(_AsyncStream): + async def __anext__(self) -> Any: raise ydb.issues.GenericError("boom") -class FakeSyncStreamSession: - def __init__(self, result_sets: list[FakeResultSet]) -> None: - self._result_sets = result_sets - self.deleted = False - - def transaction( - self, tx_mode: ydb.BaseQueryTxMode - ) -> FakeSyncStreamSession: - return self - - def execute(self, **kwargs: object) -> FakeSyncResponseContextIterator: - return FakeSyncResponseContextIterator(self._result_sets) - - def delete(self) -> None: - self.deleted = True - - -class FakeAsyncStreamSession: - def __init__(self, result_sets: list[FakeResultSet]) -> None: - self._result_sets = result_sets - self.deleted = False - - def transaction( - self, tx_mode: ydb.BaseQueryTxMode - ) -> FakeAsyncStreamSession: - return self - - async def execute( - self, **kwargs: object - ) -> FakeAsyncResponseContextIterator: - return FakeAsyncResponseContextIterator(self._result_sets) - - async def delete(self) -> None: - self.deleted = True - - -class FakeFailingSyncStreamSession(FakeSyncStreamSession): - def execute( - self, **kwargs: object - ) -> FakeFailingSyncResponseContextIterator: - return FakeFailingSyncResponseContextIterator(self._result_sets) - - -class FakeFailingAsyncStreamSession(FakeAsyncStreamSession): - async def execute( - self, **kwargs: object - ) -> FakeFailingAsyncResponseContextIterator: - return FakeFailingAsyncResponseContextIterator(self._result_sets) - - -class FakeSyncStreamSessionPool: - def __init__(self, result_sets: list[FakeResultSet]) -> None: - self.session = FakeSyncStreamSession(result_sets) - self.released_sessions: list[FakeSyncStreamSession] = [] - - def acquire(self, timeout: float | None = None) -> FakeSyncStreamSession: - return self.session - - def release(self, session: FakeSyncStreamSession) -> None: - self.released_sessions.append(session) - - -class FakeAsyncStreamSessionPool: - def __init__(self, result_sets: list[FakeResultSet]) -> None: - self.session = FakeAsyncStreamSession(result_sets) - self.released_sessions: list[FakeAsyncStreamSession] = [] - - async def acquire( - self, timeout: float | None = None - ) -> FakeAsyncStreamSession: - return self.session - - async def release(self, session: FakeAsyncStreamSession) -> None: - self.released_sessions.append(session) - - -class FakeSyncStreamConnection: - _tx_context = None - - def __init__(self) -> None: - self.current_cursor: StreamCursor | None = None - - def _invalidate_session(self) -> None: ... - - def _set_current_cursor(self, cursor: StreamCursor) -> None: - self.current_cursor = cursor - - def _clear_current_cursor(self, cursor: Cursor | None = None) -> None: - if cursor is None or self.current_cursor is cursor: - self.current_cursor = None - - -class FakeAsyncStreamConnection: - _tx_context = None - - def __init__(self) -> None: - self.current_cursor: AsyncStreamCursor | None = None - - async def _invalidate_session(self) -> None: ... - - def _set_current_cursor(self, cursor: AsyncStreamCursor) -> None: - self.current_cursor = cursor - - def _clear_current_cursor(self, cursor: AsyncCursor | None = None) -> None: - if cursor is None or self.current_cursor is cursor: - self.current_cursor = None +def _make_sync_cursor(result_sets: list) -> StreamCursor: + session = mock.MagicMock() + session.deleted = False + stream = _SyncStream(result_sets) + session.transaction.return_value.execute.return_value = stream + + def _delete() -> None: + session.deleted = True + + session.delete.side_effect = _delete + + pool = mock.MagicMock() + pool.acquire.return_value = session + released: list = [] + pool.release.side_effect = lambda s: released.append(s) + pool.released_sessions = released + pool.session = session + + connection = mock.MagicMock() + connection._tx_context = None + connection._set_current_cursor = mock.MagicMock() + connection._clear_current_cursor = mock.MagicMock() + connection._invalidate_session = mock.MagicMock() + + cursor = StreamCursor( + connection, + pool, + ydb.QuerySerializableReadWrite(), + request_settings=ydb.BaseRequestSettings(), + retry_settings=ydb.RetrySettings(), + ) + cursor._test_pool = pool + cursor._test_session = session + return cursor + + +def _make_async_cursor(result_sets: list) -> AsyncStreamCursor: + session = mock.AsyncMock() + session.deleted = False + stream = _AsyncStream(result_sets) + tx = mock.MagicMock() + tx.execute = mock.AsyncMock(return_value=stream) + session.transaction = mock.MagicMock(return_value=tx) + + async def _delete() -> None: + session.deleted = True + + session.delete.side_effect = _delete + + pool = mock.MagicMock() + pool.acquire = mock.AsyncMock(return_value=session) + released: list = [] + + async def _release(s: Any) -> None: + released.append(s) + + pool.release = mock.AsyncMock(side_effect=_release) + pool.released_sessions = released + pool.session = session + + connection = mock.MagicMock() + connection._tx_context = None + connection._set_current_cursor = mock.MagicMock() + connection._clear_current_cursor = mock.MagicMock() + connection._invalidate_session = mock.AsyncMock() + + cursor = AsyncStreamCursor( + connection, + pool, + ydb.QuerySerializableReadWrite(), + request_settings=ydb.BaseRequestSettings(), + retry_settings=ydb.RetrySettings(), + ) + cursor._test_pool = pool + cursor._test_session = session + return cursor class TestStreamCursor: def _make_cursor(self, result_sets: list) -> StreamCursor: - return StreamCursor( - FakeSyncStreamConnection(), - FakeSyncStreamSessionPool(result_sets), - ydb.QuerySerializableReadWrite(), - request_settings=ydb.BaseRequestSettings(), - retry_settings=ydb.RetrySettings(), - ) + return _make_sync_cursor(result_sets) def test_stream_cursor_fetch_one(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) try: cursor.execute("SELECT id, val FROM table") @@ -552,7 +515,7 @@ def test_stream_cursor_fetch_one(self) -> None: cursor.close() def test_stream_cursor_fetch_many(self) -> None: - cursor = self._make_cursor(make_stream_result_sets(RESULT_SET_COUNT)) + cursor = self._make_cursor(make_result_sets(RESULT_SET_COUNT)) try: cursor.execute( """ @@ -576,7 +539,7 @@ def test_stream_cursor_fetch_many(self) -> None: cursor.close() def test_stream_cursor_fetch_all(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) try: cursor.execute("SELECT id, val FROM table") @@ -588,7 +551,7 @@ def test_stream_cursor_fetch_all(self) -> None: cursor.close() def test_stream_cursor_releases_session_after_drain(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) pool = cursor._session_pool try: cursor.execute("SELECT id, val FROM table") @@ -600,7 +563,7 @@ def test_stream_cursor_releases_session_after_drain(self) -> None: cursor.close() def test_stream_cursor_discards_session_on_close(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) pool = cursor._session_pool cursor.execute("SELECT id, val FROM table") cursor.close() @@ -609,7 +572,7 @@ def test_stream_cursor_discards_session_on_close(self) -> None: assert pool.released_sessions == [pool.session] def test_stream_cursor_cancels_stream_on_close(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) cursor.execute("SELECT id, val FROM table") stream = cursor._stream cursor.close() @@ -618,9 +581,21 @@ def test_stream_cursor_cancels_stream_on_close(self) -> None: assert stream.cancelled is True def test_stream_cursor_invalidate_owner_session_on_error(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) pool = cursor._session_pool - pool.session = FakeFailingSyncStreamSession(make_stream_result_sets()) + failing_session = mock.MagicMock() + failing_session.deleted = False + failing_stream = _FailingSyncStream(make_result_sets()) + failing_session.transaction.return_value.execute.return_value = ( + failing_stream + ) + + def _delete() -> None: + failing_session.deleted = True + + failing_session.delete.side_effect = _delete + pool.session = failing_session + pool.acquire.return_value = failing_session with pytest.raises(ydb_dbapi.Error): cursor.execute("SELECT id, val FROM table") @@ -632,17 +607,11 @@ def test_stream_cursor_invalidate_owner_session_on_error(self) -> None: class TestAsyncStreamCursor: def _make_cursor(self, result_sets: list) -> AsyncStreamCursor: - return AsyncStreamCursor( - FakeAsyncStreamConnection(), - FakeAsyncStreamSessionPool(result_sets), - ydb.QuerySerializableReadWrite(), - request_settings=ydb.BaseRequestSettings(), - retry_settings=ydb.RetrySettings(), - ) + return _make_async_cursor(result_sets) @pytest.mark.asyncio async def test_stream_cursor_fetch_one(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) try: await cursor.execute("SELECT id, val FROM table") @@ -660,7 +629,7 @@ async def test_stream_cursor_fetch_one(self) -> None: @pytest.mark.asyncio async def test_stream_cursor_fetch_many(self) -> None: - cursor = self._make_cursor(make_stream_result_sets(RESULT_SET_COUNT)) + cursor = self._make_cursor(make_result_sets(RESULT_SET_COUNT)) try: await cursor.execute( """ @@ -685,7 +654,7 @@ async def test_stream_cursor_fetch_many(self) -> None: @pytest.mark.asyncio async def test_stream_cursor_fetch_all(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) try: await cursor.execute("SELECT id, val FROM table") @@ -698,7 +667,7 @@ async def test_stream_cursor_fetch_all(self) -> None: @pytest.mark.asyncio async def test_stream_cursor_discards_session_on_close(self) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) pool = cursor._session_pool await cursor.execute("SELECT id, val FROM table") await cursor.close() @@ -710,9 +679,21 @@ async def test_stream_cursor_discards_session_on_close(self) -> None: async def test_stream_cursor_invalidate_owner_session_on_error( self, ) -> None: - cursor = self._make_cursor(make_stream_result_sets()) + cursor = self._make_cursor(make_result_sets()) pool = cursor._session_pool - pool.session = FakeFailingAsyncStreamSession(make_stream_result_sets()) + failing_session = mock.AsyncMock() + failing_session.deleted = False + failing_stream = _FailingAsyncStream(make_result_sets()) + tx = mock.MagicMock() + tx.execute = mock.AsyncMock(return_value=failing_stream) + failing_session.transaction = mock.MagicMock(return_value=tx) + + async def _delete() -> None: + failing_session.deleted = True + + failing_session.delete.side_effect = _delete + pool.session = failing_session + pool.acquire = mock.AsyncMock(return_value=failing_session) with pytest.raises(ydb_dbapi.Error): await cursor.execute("SELECT id, val FROM table") diff --git a/ydb_dbapi/connections.py b/ydb_dbapi/connections.py index 7ca91b5..f124efb 100644 --- a/ydb_dbapi/connections.py +++ b/ydb_dbapi/connections.py @@ -308,6 +308,7 @@ def wait_ready(self, timeout: int = 10) -> None: @handle_ydb_errors def begin(self) -> None: + self._raise_if_current_cursor_running() self._tx_context = None if self.interactive_transaction: self._session = self._session_pool.acquire() @@ -564,6 +565,7 @@ async def wait_ready(self, timeout: int = 10) -> None: @handle_ydb_errors async def begin(self) -> None: + self._raise_if_current_cursor_running() self._tx_context = None if self.interactive_transaction: self._session = await self._session_pool.acquire() diff --git a/ydb_dbapi/cursors.py b/ydb_dbapi/cursors.py index 24a12e5..3faa284 100644 --- a/ydb_dbapi/cursors.py +++ b/ydb_dbapi/cursors.py @@ -344,6 +344,8 @@ def execute_scheme( ) -> None: self._raise_if_closed() self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) ydb_parameters = _as_ydb_parameters(parameters) @@ -658,6 +660,7 @@ def close(self) -> None: if self._state == CursorStatus.running: if self._session_owner is not None and self._stream is not None: self._stream.cancel() + self._finalize_rowcount() self._stream = None self._discard_owned_session() elif self._stream is not None: @@ -801,6 +804,8 @@ async def execute_scheme( ) -> None: self._raise_if_closed() self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) ydb_parameters = _as_ydb_parameters(parameters) @@ -944,12 +949,19 @@ async def execute_scheme( ) -> None: self._raise_if_closed() self._raise_if_running() + if self._tx_context is not None: + self._connection._raise_if_current_cursor_running() query = self._append_table_path_prefix(query) + + if self._pyformat and parameters is not None: + query, parameters = convert_query_parameters(query, parameters) + + ydb_parameters = _as_ydb_parameters(parameters) self._begin_query() result_list = await self._execute_generic_query( - query=query, parameters=parameters + query=query, parameters=ydb_parameters ) self._fill_buffer(result_list) self._finish_query() @@ -1183,6 +1195,7 @@ async def close(self) -> None: if self._state == CursorStatus.running: if self._session_owner is not None and self._stream is not None: self._stream.cancel() + self._finalize_rowcount() self._stream = None await self._discard_owned_session() elif self._stream is not None: From 79e4a95bd11033ce5d2748c16766a766101e8ee2 Mon Sep 17 00:00:00 2001 From: GrigorySmirnovSpb Date: Fri, 6 Nov 2026 01:20:29 +0300 Subject: [PATCH 4/4] Fix Copilot's remarks --- CHANGELOG.md | 5 ----- ydb_dbapi/connections.py | 20 ++++++++++++++++++++ ydb_dbapi/cursors.py | 19 +++++++++++++------ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dafe4de..7564a24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,3 @@ -## 0.1.25 ## -* Add stream cursors (server-side cursor): connection.cursor(stream_results=True) -* Guard transactional operations while a stream cursor is streaming -* Sync and async stream cursors with lazy result-set fetching - ## 0.1.24 ## * Fix DBAPI typing with newer YDB SDK releases diff --git a/ydb_dbapi/connections.py b/ydb_dbapi/connections.py index f124efb..3e7fe8b 100644 --- a/ydb_dbapi/connections.py +++ b/ydb_dbapi/connections.py @@ -272,6 +272,7 @@ def __init__( **kwargs, ) self._current_cursor: Cursor | StreamCursor | None = None + self._active_stream_cursors: set[StreamCursor] = set() def _create_session_pool( self, driver_config: ydb.DriverConfig @@ -350,6 +351,9 @@ def _rollback(self) -> None: @handle_ydb_errors def close(self) -> None: + for cursor in list(self._active_stream_cursors): + cursor.close() + self._active_stream_cursors.clear() if self._current_cursor is not None: self._current_cursor.close() try: @@ -474,6 +478,12 @@ def _clear_current_cursor(self, cursor: Cursor | None = None) -> None: if cursor is None or self._current_cursor is cursor: self._current_cursor = None + def _register_stream_cursor(self, cursor: StreamCursor) -> None: + self._active_stream_cursors.add(cursor) + + def _unregister_stream_cursor(self, cursor: StreamCursor) -> None: + self._active_stream_cursors.discard(cursor) + def _raise_if_current_cursor_running( self, cursor: Cursor | None = None ) -> None: @@ -527,6 +537,7 @@ def __init__( **kwargs, ) self._current_cursor: AsyncStreamCursor | None = None + self._active_stream_cursors: set[AsyncStreamCursor] = set() def _create_session_pool( self, driver_config: ydb.DriverConfig @@ -607,6 +618,9 @@ async def _rollback(self) -> None: @handle_ydb_errors async def close(self) -> None: + for cursor in list(self._active_stream_cursors): + await cursor.close() + self._active_stream_cursors.clear() if self._current_cursor is not None: await self._current_cursor.close() try: @@ -737,6 +751,12 @@ def _clear_current_cursor( if cursor is None or self._current_cursor is cursor: self._current_cursor = None + def _register_stream_cursor(self, cursor: AsyncStreamCursor) -> None: + self._active_stream_cursors.add(cursor) + + def _unregister_stream_cursor(self, cursor: AsyncStreamCursor) -> None: + self._active_stream_cursors.discard(cursor) + def _raise_if_current_cursor_running( self, cursor: AsyncStreamCursor | None = None ) -> None: diff --git a/ydb_dbapi/cursors.py b/ydb_dbapi/cursors.py index 3faa284..dffa8c4 100644 --- a/ydb_dbapi/cursors.py +++ b/ydb_dbapi/cursors.py @@ -442,6 +442,7 @@ def __init__( def _invalidate_active_session(self) -> None: self._clear_current_cursor() + self._stream = None if self._session_owner is None: self._connection._invalidate_session() return @@ -456,10 +457,12 @@ def _invalidate_active_session(self) -> None: def _clear_current_cursor(self) -> None: self._connection._clear_current_cursor(self) + self._connection._unregister_stream_cursor(self) def _register_current_cursor(self) -> None: if self._tx_context is not None: self._connection._set_current_cursor(self) + self._connection._register_stream_cursor(self) def _release_owned_session(self) -> None: if self._session_owner is None: @@ -618,6 +621,7 @@ def execute( query=query, parameters=ydb_parameters, ) + self._register_current_cursor() self._prime_stream() @@ -966,12 +970,6 @@ async def execute_scheme( self._fill_buffer(result_list) self._finish_query() - async def executemany( - self, query: str, seq_of_parameters: Sequence[ParametersType] - ) -> None: - for parameters in seq_of_parameters: - await self.execute(query, parameters) - async def _invalidate_active_session(self) -> None: self._clear_current_cursor() self._stream = None @@ -988,10 +986,12 @@ async def _invalidate_active_session(self) -> None: def _clear_current_cursor(self) -> None: self._connection._clear_current_cursor(self) + self._connection._unregister_stream_cursor(self) def _register_current_cursor(self) -> None: if self._tx_context is not None: self._connection._set_current_cursor(self) + self._connection._register_stream_cursor(self) async def _release_owned_session(self) -> None: if self._session_owner is None: @@ -1120,6 +1120,12 @@ async def _execute_transactional_query_stream( settings=settings, ) + async def executemany( + self, query: str, seq_of_parameters: Sequence[ParametersType] + ) -> None: + for parameters in seq_of_parameters: + await self.execute(query, parameters) + async def execute( self, query: str, @@ -1150,6 +1156,7 @@ async def execute( query=query, parameters=ydb_parameters, ) + self._register_current_cursor() await self._prime_stream()