Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [3.9]
python-version: ['3.9', '3.12']
poetry-version: [1.8]

steps:
Expand Down
13 changes: 11 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ async_connection = await ydb_dbapi.async_connect(
) # async connection
```

For TLS connections, pass PEM certificate contents as `bytes` through
`root_certificates`, or use `root_certificates_path` to load them from a file.
Shared session pools must match the connection API: `ydb.QuerySessionPool`
for `connect()` and `ydb.aio.QuerySessionPool` for `async_connect()`.

Usage of connection:

```python
Expand Down Expand Up @@ -138,8 +143,12 @@ YDB type.
> `pyformat=True` at your earliest convenience.

By default (`pyformat=False`) the driver passes the query and parameters
directly to the YDB SDK without any transformation. Use `$name` placeholders
in the query and supply a `dict` with `$`-prefixed keys:
directly to the YDB SDK without any transformation.
Passing a `list` or `tuple` in native mode raises `ProgrammingError`.
Use `pyformat=True` with `execute()` for positional parameters.

Use `$name` placeholders in the query and supply a `dict` with
`$`-prefixed keys:

```python
connection = ydb_dbapi.connect(
Expand Down
2,531 changes: 1,554 additions & 977 deletions poetry.lock

Large diffs are not rendered by default.

18 changes: 18 additions & 0 deletions tests/test_connect_options.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,29 @@
from __future__ import annotations

import pytest
import ydb
import ydb_dbapi as dbapi
from ydb_dbapi.errors import ProgrammingError
from ydb_dbapi.utils import prepare_credentials
from ydb_dbapi.utils import prepare_driver_config_kwargs


@pytest.mark.parametrize(
("credentials", "password"),
[
({"username": "root"}, ""),
({"username": "root", "password": None}, ""),
({"username": "root", "password": ""}, ""),
({"username": "root", "password": "secret"}, "secret"),
],
)
def test_static_credentials_password(credentials: dict, password: str):
result = prepare_credentials(credentials)
assert isinstance(result, ydb.StaticCredentials)
assert result.user == "root"
assert result.password == password


class TestPrepareDriverConfigKwargs:
"""Leftover connect() keywords are routed, not dropped."""

Expand Down
74 changes: 52 additions & 22 deletions tests/test_connections.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ def _test_commit_rollback_after_begin(
maybe_await(connection.begin())
maybe_await(connection.rollback())


def _test_close_releases_session_after_rollback_error(
self,
connection: dbapi.Connection,
Expand Down Expand Up @@ -208,9 +207,12 @@ def _test_cursor_raw_query(self, connection: dbapi.Connection) -> None:
with suppress(dbapi.DatabaseError):
maybe_await(cur.execute_scheme("DROP TABLE test"))

maybe_await(cur.execute_scheme(
"CREATE TABLE test(id Int64 NOT NULL, text Utf8, PRIMARY KEY (id))"
))
maybe_await(
cur.execute_scheme(
"CREATE TABLE test("
"id Int64 NOT NULL, text Utf8, PRIMARY KEY (id))"
)
)

maybe_await(
cur.execute(
Expand Down Expand Up @@ -322,7 +324,24 @@ def _test_errors(
maybe_await(cur.execute_scheme("DROP TABLE test"))
maybe_await(cur.close())

def _test_bulk_upsert(self, connection: dbapi.Connection) -> None:
def _test_native_parameters_require_dict(
self, connection: dbapi.Connection
) -> None:
cursor = connection.cursor()
for method in (cursor.execute, cursor.execute_scheme):
for parameters in ([1], (1,), [], ()):
with pytest.raises(
dbapi.ProgrammingError, match="parameters must be a dict"
):
maybe_await(method("SELECT 1", parameters))

maybe_await(cursor.execute("SELECT 1"))
assert cursor.fetchone() == (1,)
maybe_await(cursor.close())

def _test_bulk_upsert(
self, connection: dbapi.Connection, rows_type: type
) -> None:
cursor = connection.cursor()
with suppress(dbapi.DatabaseError):
maybe_await(cursor.execute_scheme("DROP TABLE pet"))
Expand Down Expand Up @@ -368,7 +387,9 @@ def _test_bulk_upsert(self, connection: dbapi.Connection) -> None:
},
]

maybe_await(connection.bulk_upsert("pet", rows, column_types))
maybe_await(
connection.bulk_upsert("pet", rows_type(rows), column_types)
)

maybe_await(cursor.execute("SELECT * FROM pet"))
assert cursor.rowcount == 2
Expand Down Expand Up @@ -589,9 +610,7 @@ def test_commit_rollback_after_begin(
isolation_level: str,
connection: dbapi.Connection,
) -> None:
self._test_commit_rollback_after_begin(
connection, isolation_level
)
self._test_commit_rollback_after_begin(connection, isolation_level)

def test_close_releases_session_after_rollback_error(
self, connection: dbapi.Connection
Expand Down Expand Up @@ -624,22 +643,26 @@ def test_cursor_pyformat_query(self, connection_kwargs: dict) -> None:
def test_errors(self, connection: dbapi.Connection) -> None:
self._test_errors(connection)

def test_bulk_upsert(self, connection: dbapi.Connection) -> None:
self._test_bulk_upsert(connection)
def test_native_parameters_require_dict(
self, connection: dbapi.Connection
) -> None:
self._test_native_parameters_require_dict(connection)

@pytest.mark.parametrize("rows_type", [list, tuple])
def test_bulk_upsert(
self, connection: dbapi.Connection, rows_type: type
) -> None:
self._test_bulk_upsert(connection, rows_type)

def test_errors_with_interactive_tx(
self, connection: dbapi.Connection
) -> None:
self._test_error_with_interactive_tx(connection)

def test_get_view_names(
self, connection: dbapi.Connection
) -> None:
def test_get_view_names(self, connection: dbapi.Connection) -> None:
self._test_get_view_names(connection)

def test_get_table_names(
self, connection: dbapi.Connection
) -> None:
def test_get_table_names(self, connection: dbapi.Connection) -> None:
self._test_get_table_names(connection)


Expand Down Expand Up @@ -717,9 +740,7 @@ async def test_commit_rollback_after_begin(
connection: dbapi.AsyncConnection,
) -> None:
await greenlet_spawn(
self._test_commit_rollback_after_begin,
connection,
isolation_level
self._test_commit_rollback_after_begin, connection, isolation_level
)

@pytest.mark.asyncio
Expand Down Expand Up @@ -774,10 +795,19 @@ async def test_errors(self, connection: dbapi.AsyncConnection) -> None:
await greenlet_spawn(self._test_errors, connection)

@pytest.mark.asyncio
async def test_bulk_upsert(
async def test_native_parameters_require_dict(
self, connection: dbapi.AsyncConnection
) -> None:
await greenlet_spawn(self._test_bulk_upsert, connection)
await greenlet_spawn(
self._test_native_parameters_require_dict, connection
)

@pytest.mark.asyncio
@pytest.mark.parametrize("rows_type", [list, tuple])
async def test_bulk_upsert(
self, connection: dbapi.AsyncConnection, rows_type: type
) -> None:
await greenlet_spawn(self._test_bulk_upsert, connection, rows_type)

@pytest.mark.asyncio
async def test_errors_with_interactive_tx(
Expand Down
4 changes: 1 addition & 3 deletions tests/test_convert_parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,7 @@ def test_multiple_params(self):
assert "$c" in p

def test_percent_percent_escape(self):
q, _ = convert_query_parameters(
"SELECT %% as pct, %(x)s", {"x": 1}
)
q, _ = convert_query_parameters("SELECT %% as pct, %(x)s", {"x": 1})
assert q == "SELECT % as pct, $x"

def test_empty_params(self):
Expand Down
63 changes: 47 additions & 16 deletions ydb_dbapi/connections.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
from __future__ import annotations

import posixpath
from abc import ABC
from abc import abstractmethod
from collections.abc import Sequence
from enum import Enum
from typing import Any
Expand Down Expand Up @@ -68,10 +70,7 @@ class _IsolationSettings(NamedTuple):
}


class BaseConnection:
_driver_cls = ydb.Driver
_pool_cls = ydb.QuerySessionPool

class BaseConnection(ABC):
def __init__(
self,
host: str = "",
Expand All @@ -82,7 +81,7 @@ def __init__(
credentials: ydb.Credentials | dict | str | None = None,
ydb_session_pool: SessionPool | AsyncSessionPool | None = None,
root_certificates_path: str | None = None,
root_certificates: str | None = None,
root_certificates: bytes | None = None,
driver_config_kwargs: dict | None = None,
pyformat: bool = False,
**kwargs: Any,
Expand Down Expand Up @@ -157,15 +156,21 @@ def __init__(
_additional_sdk_headers=framework_headers,
**driver_config_kwargs,
)
self._driver = self._driver_cls(driver_config)
self._session_pool = self._pool_cls(self._driver, size=5)
self._session_pool = self._create_session_pool(driver_config)
self._driver = self._session_pool._driver

self._session: ydb.QuerySession | ydb.aio.QuerySession | None = None
self.request_settings: ydb.BaseRequestSettings = (
ydb.BaseRequestSettings()
)
self.retry_settings: ydb.RetrySettings = ydb.RetrySettings()

@abstractmethod
def _create_session_pool(
self, driver_config: ydb.DriverConfig
) -> SessionPool | AsyncSessionPool:
raise NotImplementedError

def set_isolation_level(self, isolation_level: IsolationLevel) -> None:
if self._tx_context and self._tx_context.tx_id:
raise InternalError(
Expand All @@ -182,7 +187,7 @@ def get_isolation_level(self) -> str:
if self.interactive_transaction:
return IsolationLevel.SERIALIZABLE
return IsolationLevel.AUTOCOMMIT
if self._tx_mode.name == ydb.QueryOnlineReadOnly().name:
if isinstance(self._tx_mode, ydb.QueryOnlineReadOnly):
if self._tx_mode.allow_inconsistent_reads:
return IsolationLevel.ONLINE_READONLY_INCONSISTENT
return IsolationLevel.ONLINE_READONLY
Expand Down Expand Up @@ -228,6 +233,10 @@ class Connection(BaseConnection):
_driver_cls = ydb.Driver
_pool_cls = ydb.QuerySessionPool
_cursor_cls = Cursor
_driver: ydb.Driver
_session_pool: SessionPool
_session: ydb.QuerySession | None
_tx_context: TxContext | None

def __init__(
self,
Expand All @@ -237,9 +246,9 @@ def __init__(
ydb_table_path_prefix: str = "",
protocol: str | None = None,
credentials: ydb.Credentials | dict | str | None = None,
ydb_session_pool: SessionPool | AsyncSessionPool | None = None,
ydb_session_pool: SessionPool | None = None,
root_certificates_path: str | None = None,
root_certificates: str | None = None,
root_certificates: bytes | None = None,
driver_config_kwargs: dict | None = None,
pyformat: bool = False,
**kwargs: Any,
Expand All @@ -260,6 +269,12 @@ def __init__(
)
self._current_cursor: Cursor | None = None

def _create_session_pool(
self, driver_config: ydb.DriverConfig
) -> SessionPool:
driver = self._driver_cls(driver_config)
return self._pool_cls(driver, size=5)

def cursor(self) -> Cursor:
return self._cursor_cls(
connection=self,
Expand Down Expand Up @@ -410,6 +425,9 @@ def bulk_upsert(
self.database, self.table_path_prefix, table_name
)

if not isinstance(rows, list):
rows = list(rows)

self._driver.table_client.bulk_upsert(
abs_table_path,
rows=rows,
Expand All @@ -429,6 +447,10 @@ class AsyncConnection(BaseConnection):
_driver_cls = ydb.aio.Driver
_pool_cls = ydb.aio.QuerySessionPool
_cursor_cls = AsyncCursor
_driver: ydb.aio.Driver
_session_pool: AsyncSessionPool
_session: ydb.aio.QuerySession | None
_tx_context: AsyncTxContext | None

def __init__(
self,
Expand All @@ -438,9 +460,9 @@ def __init__(
ydb_table_path_prefix: str = "",
protocol: str | None = None,
credentials: ydb.Credentials | dict | str | None = None,
ydb_session_pool: SessionPool | AsyncSessionPool | None = None,
ydb_session_pool: AsyncSessionPool | None = None,
root_certificates_path: str | None = None,
root_certificates: str | None = None,
root_certificates: bytes | None = None,
driver_config_kwargs: dict | None = None,
pyformat: bool = False,
**kwargs: Any,
Expand All @@ -461,6 +483,12 @@ def __init__(
)
self._current_cursor: AsyncCursor | None = None

def _create_session_pool(
self, driver_config: ydb.DriverConfig
) -> AsyncSessionPool:
driver = self._driver_cls(driver_config)
return self._pool_cls(driver, size=5)

def cursor(self) -> AsyncCursor:
return self._cursor_cls(
connection=self,
Expand Down Expand Up @@ -615,6 +643,9 @@ async def bulk_upsert(
self.database, self.table_path_prefix, table_name
)

if not isinstance(rows, list):
rows = list(rows)

await self._driver.table_client.bulk_upsert(
abs_table_path,
rows=rows,
Expand All @@ -637,9 +668,9 @@ def connect(
ydb_table_path_prefix: str = "",
protocol: str | None = None,
credentials: ydb.Credentials | dict | str | None = None,
ydb_session_pool: SessionPool | AsyncSessionPool | None = None,
ydb_session_pool: SessionPool | None = None,
root_certificates_path: str | None = None,
root_certificates: str | None = None,
root_certificates: bytes | None = None,
driver_config_kwargs: dict | None = None,
pyformat: bool = False,
**kwargs: Any,
Expand Down Expand Up @@ -669,9 +700,9 @@ async def async_connect(
ydb_table_path_prefix: str = "",
protocol: str | None = None,
credentials: ydb.Credentials | dict | str | None = None,
ydb_session_pool: SessionPool | AsyncSessionPool | None = None,
ydb_session_pool: AsyncSessionPool | None = None,
root_certificates_path: str | None = None,
root_certificates: str | None = None,
root_certificates: bytes | None = None,
driver_config_kwargs: dict | None = None,
pyformat: bool = False,
**kwargs: Any,
Expand Down
Loading
Loading