diff --git a/README.md b/README.md index 8ded072f..c77a32f4 100644 --- a/README.md +++ b/README.md @@ -136,13 +136,13 @@ dialect or backend you use: | Local/S3 Iceberg and Lance | Ray Data / Daft public readers | `tributo[data,data-daft]` | Alpha; real dual-engine Conformance | | PostgreSQL structured table | Ray Data / Daft SQL readers | `tributo[postgresql,data-daft]` | Alpha; real PostgreSQL Conformance | | HDFS Parquet/CSV | Ray Data + PyArrow Hadoop filesystem | Ray runtime with HDFS libraries | Adapter present; cluster gate pending | -| ClickHouse | `ray-clickhouse==0.1.0` / `daft-clickhouse==1.0` | `tributo[clickhouse]` plus the external Ray wheel when selecting Ray | Adapter only; the full image contains the Daft package, while the Ray package uses the external wheelhouse until PyPI publication and real-database Conformance remains the support gate | +| ClickHouse | `ray-clickhouse==0.1.0` external wheel | `tributo[clickhouse]` plus the external Ray wheel | Adapter only; the Ray package uses the external wheelhouse until PyPI publication and real-database Conformance remains the support gate | | Doris | `ray-doris==1.0` / `daft-doris==1.0` | `tributo[mysql]` or `tributo[doris-flight]` | Adapter only; Ray routes require `ray-doris`, Daft routes require `daft-doris`, and real-database Conformance remains the support gate | | HiveServer2 structured tables | `ray-hive==1.0` through the built-in Ray Binding | `tributo[hive-ray]` or `uv sync --locked --no-dev --extra hive-ray` | Alpha; real Hive 4.2.0 structured-projection Conformance | | Native ORC files | no built-in reader | — | Unsupported; use an engine or table service that owns ORC decoding | Provider/binding presence is not a support claim. The canonical full image -contains the locked v1.0 Daft ClickHouse and Doris connector packages, but the +contains the locked v1.0 Doris connector packages, but the Ray ClickHouse route requires the external `ray-clickhouse==0.1.0` wheel until that package is published to PyPI. These paths remain adapter-only until their real-database Conformance gates pass. diff --git a/docs/STABILITY.md b/docs/STABILITY.md index 44ad7a7b..c91ecf2d 100644 --- a/docs/STABILITY.md +++ b/docs/STABILITY.md @@ -44,7 +44,7 @@ covered by the external `runtime-image` suite. | Surface | Level | Notes | |--------|-------|-------| | `tools/build_tributo_image.py` | `alpha` | JSON-only Buildx builder for the pinned image validated for CPU execution by default, with explicit `linux/amd64`/`linux/arm64` targeting; it performs dependency-closure discovery, manifest sealing, and fail-closed import checks | -| `tools/tributo-runtime-full.json` | `alpha` | Multi-architecture pinned Ray/uv image references, native-platform default, complete first-party runtime-extra closure including locked v1.0 Daft ClickHouse/Doris and Ray Doris connectors plus `ray-hive`; optional external wheelhouse support supplies `ray-clickhouse==0.1.0` until PyPI publication | +| `tools/tributo-runtime-full.json` | `alpha` | Multi-architecture pinned Ray/uv image references, native-platform default, complete first-party runtime-extra closure including locked v1.0 Daft Doris and Ray Doris connectors plus `ray-hive`; optional external wheelhouse support supplies `ray-clickhouse==0.1.0` until PyPI publication | | `manifest.json` / `image-profile.json` | `alpha` | Build attestations consumed for immutable image selection and algorithm artifact compatibility; not a registry or deployment API | ### Training (tributo.training.*) diff --git a/docs/architecture/call-chain-inventory.md b/docs/architecture/call-chain-inventory.md index 4b9c9678..d7807cbb 100644 --- a/docs/architecture/call-chain-inventory.md +++ b/docs/architecture/call-chain-inventory.md @@ -301,9 +301,9 @@ WriteBinding selection; native dependency import occurs at factory/execute ``` Selected optional integrations (`ray-clickhouse==0.1.0`, `ray-doris==1.0`, -`daft-doris==1.0`, `daft-clickhouse==1.0`) also have thin built-in descriptors -and explicit install diagnostics. The canonical full runtime locks the v1.0 -packages into the image; `ray-clickhouse` remains an external-wheelhouse input +`daft-doris==1.0`) also have thin built-in descriptors and explicit install +diagnostics. The canonical full runtime locks the v1.0 Doris packages into the +image; `ray-clickhouse` remains an external-wheelhouse input until PyPI publication. These adapters are not support claims until database infrastructure gates pass. Ray and Daft routes remain explicit and are not interchangeable. diff --git a/docs/data/index.md b/docs/data/index.md index d5eb548f..bf4ff50b 100644 --- a/docs/data/index.md +++ b/docs/data/index.md @@ -56,7 +56,7 @@ selects `binding_id` explicitly. | Local/S3 Lance | Native reader | Native reader | Verified | | PostgreSQL structured table | Native SQL reader | Native SQL reader | Verified | | HDFS Parquet/CSV | Native reader with PyArrow HDFS | No locked public reader | Adapted; cluster gate pending | -| ClickHouse | `ray-clickhouse==0.1.0` external wheel | `daft-clickhouse==1.0` | Adapter only; Ray uses the installed wheel through `ray_clickhouse.ray.clickhouse`, Daft uses `tributo[clickhouse]`, and real-database Conformance remains the support gate | +| ClickHouse | `ray-clickhouse==0.1.0` external wheel | No built-in route | Adapter only; Ray uses the installed wheel through `ray_clickhouse.ray.clickhouse`, and real-database Conformance remains the support gate | | Doris | `ray-doris==1.0` | `daft-doris==1.0` | Adapter only; Ray routes use `ray-doris`, Daft routes use `daft-doris`, and the full runtime image contains both v1.0 packages | | HiveServer2 structured table | `ray-hive==1.0` through `tributo.ray.hive` | No built-in route | Alpha, verified with Hive 4.2.0; projection and worker execution only through binary HiveServer2 | | Native ORC file | No built-in route | No built-in route | Unsupported; the HiveServer2 path does not expose ORC/HDFS files | @@ -64,10 +64,10 @@ selects `binding_id` explicitly. “Verified” means the current combination has semantic Conformance and real storage or database evidence. It does not turn every engine/source combination into a supported path. See the [support matrix](../reference/support-matrix.md) -for the exact boundary. Ray ClickHouse, Daft ClickHouse, Daft Doris, and Ray -Doris are explicit engine routes; a missing wheel never causes an engine -fallback. Hive requests must explicitly select the Ray engine; Tributo does not -add a Daft route or fall back to file-level ORC/HDFS access. +for the exact boundary. Ray ClickHouse, Daft Doris, and Ray Doris are explicit +engine routes; a missing wheel never causes an engine fallback. Hive requests +must explicitly select the Ray engine; Tributo does not add a Daft route or fall +back to file-level ORC/HDFS access. Credentials belong to runtime configuration. They must not appear in dataset identifiers, logical plans, receipts, logs, or public errors. Bounded input diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index ab310b9e..06db0d33 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -29,7 +29,6 @@ Pydantic, ONNX Runtime, PyArrow, pandas, and S3 filesystem support. | Daft ingestion | `uv sync --locked --no-dev --extra data --extra data-daft` | | HiveServer2 via Ray Data connector package | `uv sync --locked --no-dev --extra hive-ray` | | PostgreSQL ingestion | `uv sync --locked --no-dev --extra postgresql` | -| ClickHouse via Daft | `uv sync --locked --no-dev --extra clickhouse` | | ClickHouse via Ray Data | Sync `clickhouse`, then install the external `ray-clickhouse==0.1.0` wheel into `.venv` | | Doris via Daft/Ray Data | `uv sync --locked --no-dev --extra mysql` | | Doris Flight via Daft/Ray Data | `uv sync --locked --no-dev --extra doris-flight` | @@ -50,10 +49,10 @@ An extra installs dependencies. It does not turn a protocol, adapter, or reserved problem type into a verified implementation. Check the [support matrix](../reference/support-matrix.md) before deployment. Ray Tune itself is included by the core Ray dependency; the `tune` extra adds the -optional BayesOpt search implementation. The `clickhouse` extra installs -`daft-clickhouse==1.0` and the shared ClickHouse driver dependencies. The Ray -route additionally requires the `ray-clickhouse==0.1.0` GitHub Release wheel -until its PyPI publication. `mysql` installs `daft-doris==1.0` and +optional BayesOpt search implementation. The `clickhouse` extra installs the +shared ClickHouse driver dependency. ClickHouse reads use the Ray route, which +additionally requires the `ray-clickhouse==0.1.0` GitHub Release wheel until its +PyPI publication. `mysql` installs `daft-doris==1.0` and `ray-doris==1.0` for their explicit engine routes, while `doris-flight` adds their Flight dependencies. The `hive-ray` extra installs `ray-hive==1.0` for the built-in Ray-only HiveServer2 Provider/Binding route. This does not add diff --git a/docs/how-to/runtime-images.md b/docs/how-to/runtime-images.md index 6591029b..db41b01f 100644 --- a/docs/how-to/runtime-images.md +++ b/docs/how-to/runtime-images.md @@ -60,10 +60,9 @@ The output directory contains: - `build.log`: the command/evidence log for this build. The builder never pushes an image. Publishing requires a separate reviewed -workflow. The canonical configuration resolves `daft-clickhouse==1.0`, -`daft-doris==1.0`, `ray-doris==1.0`, and `ray-hive==1.0` through the locked -Tributo extras, so the normal image build does not require a local connector -wheelhouse. The +workflow. The canonical configuration resolves `daft-doris==1.0`, +`ray-doris==1.0`, and `ray-hive==1.0` through the locked Tributo extras, so the +normal image build does not require a local connector wheelhouse. The `external_wheelhouse` option remains available for packages outside the lock: it is copied into a named build context and installed with `pip --no-index --no-deps`; every wheel is recorded by filename, diff --git a/docs/reference/api/data.md b/docs/reference/api/data.md index 220bc8ce..8175166f 100644 --- a/docs/reference/api/data.md +++ b/docs/reference/api/data.md @@ -11,10 +11,6 @@ documentation for every public stability tier. ## `tributo.data.bindings._daft_sql` -```{autoclass} tributo.data.bindings._daft_sql.DaftClickHouseBinding -:no-members: -``` - ```{autoclass} tributo.data.bindings._daft_sql.DaftDorisBinding :no-members: ``` diff --git a/docs/reference/support-matrix.md b/docs/reference/support-matrix.md index 47c2383f..5f37f379 100644 --- a/docs/reference/support-matrix.md +++ b/docs/reference/support-matrix.md @@ -49,7 +49,7 @@ endpoint. | PostgreSQL structured table reads | Verified | Ray uses a single public SQL read and fails closed on parallel shard requirements; Daft may use native partition hints | | ClickHouse/Doris raw SQL | Unsupported | Legacy shapes return a credential-free migration error; use structured table input or execute SQL outside Tributo ingestion | | HDFS Parquet/CSV reads | Adapter only | Ray binding exists; real HDFS/JVM/worker gate is pending | -| ClickHouse reads | Adapter only | Daft uses locked `daft-clickhouse==1.0`; Ray uses the separately installed `ray-clickhouse==0.1.0` wheel and delegates single, physical-partition, integer-range, and bounded Arrow block behavior to it. Real-database Tributo Conformance is still required | +| ClickHouse reads | Adapter only | Ray uses the separately installed `ray-clickhouse==0.1.0` wheel and delegates single, physical-partition, integer-range, and bounded Arrow block behavior to it. Real-database Tributo Conformance is still required | | Doris reads | Adapter only | Ray routes use locked `ray-doris==1.0`; Daft routes use locked `daft-doris==1.0`; real-database Conformance is still required and tablet planning remains provider/binding-owned | | HiveServer2 structured table reads | Alpha, verified | `ProviderSourceConfig(provider="tributo.hive", uri="hive://host:port/database/table")` routes only to the Ray `tributo.ray.hive` Binding and locked `ray-hive==1.0`; Hive 4.2.0 Conformance verifies projection, schema, worker execution, and binary NOSASL transport. Raw SQL, Daft Hive, HTTP, TLS, Kerberos, and Hive writes are outside this contract | | Native ORC file reads | Unsupported | Tributo does not implement an ORC reader. The verified Hive fixture is ORC-backed, but HiveServer2 and `ray-hive` own its SQL execution and decoding; this is not HDFS or file-level ORC support | @@ -173,7 +173,7 @@ tensors; it does not apply DNN/PU preprocessing implicitly. | Full Tributo runtime image for CPU validation | Alpha, directly buildable | Linux `arm64`/`amd64`, defaulting to the native host architecture; Python 3.12, Ray 2.55.1, locked dependency closure, and all first-party runtime extras including Alpha modules. Linux PyTorch resolution may include transitive CUDA/NVIDIA distributions, but no GPU support is claimed | | Custom connector wheelhouse variant | Alpha, optional extension | An external wheelhouse remains available for packages outside the locked connector set and is currently required for `ray-clickhouse==0.1.0`; it does not change the Tributo lockfile | | Runtime image attestation | Alpha | `manifest.json`, `image-profile.json`, normalized distribution inventory, and sealed `org.tributo.manifest-sha256` label | -| Runtime image Ray Jobs gate | Alpha, validation gate | Requires a unique two-node Docker Ray cluster on a native host architecture; verifies driver/worker imports, Ray Data, Jobs API submission, and v1.0 ClickHouse/Doris/Ray Hive package presence. `linux/amd64` requires a matching native host | +| Runtime image Ray Jobs gate | Alpha, validation gate | Requires a unique two-node Docker Ray cluster on a native host architecture; verifies driver/worker imports, Ray Data, Jobs API submission, and the ClickHouse driver plus v1.0 Doris/Ray Hive connector package presence. `linux/amd64` requires a matching native host | | GPU runtime image | Not implemented | The Linux dependency closure may contain transitive CUDA/NVIDIA distributions from PyTorch, but no GPU driver, scheduling, NCCL, or GPU compatibility contract has been validated | | HiveServer2 Runtime | Alpha, validation gate | `ray-hive==1.0` is locked into the full image and selected by the built-in Ray Hive Binding; the Data Ingestion Gate verifies Hive 4.2.0 on a worker. HDFS and native ORC readers remain outside this runtime contract | diff --git a/pyproject.toml b/pyproject.toml index 3e0c9182..05e2a10e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,6 @@ postgresql = [ ] clickhouse = [ "clickhouse-connect[arrow,async]>=1.5,<1.6", - "daft-clickhouse[clickhouse]==1.0", ] mysql = [ "PyMySQL>=1.2,<1.3", diff --git a/src/tributo/data/bindings/__init__.py b/src/tributo/data/bindings/__init__.py index 4ed93a73..e768bcd8 100644 --- a/src/tributo/data/bindings/__init__.py +++ b/src/tributo/data/bindings/__init__.py @@ -27,10 +27,6 @@ _DAFT_INSTALL_HINT = "pip install 'tributo[data-daft]'" _DAFT_LANCE_INSTALL_HINT = "pip install 'tributo[data,data-daft]'" _DATA_INSTALL_HINT = "pip install 'tributo[data]'" -_DAFT_CLICKHOUSE_INSTALL_HINT = ( - "Install daft-clickhouse==1.0 with Tributo: " - "pip install 'tributo[clickhouse]' (or uv sync --extra clickhouse)" -) _RAY_CLICKHOUSE_INSTALL_HINT = ( "Install the ray-clickhouse==0.1.0 wheel, then install Tributo's " "ClickHouse dependencies with pip install 'tributo[clickhouse]'" @@ -281,29 +277,6 @@ def _daft_lance_descriptor() -> BindingDescriptor: ) -def _daft_clickhouse_descriptor() -> BindingDescriptor: - from tributo.data.bindings.daft_clickhouse import DaftClickHouseBinding - - return BindingDescriptor( - key=BindingKey( - "tributo.daft", - ScanKind.SQL, - "clickhouse", - "daft_clickhouse.daft.clickhouse", - ), - factory=DaftClickHouseBinding, - capabilities=frozenset({SourceCapability.PROJECTION}), - distribution_name="daft-clickhouse", - distribution_version=_distribution_version("daft-clickhouse") or "1.0", - engine_version_spec=_DAFT_SQL_VERSION_SPEC, - dependency_distributions=("clickhouse-connect",), - supported_read_hints=frozenset( - {ReadHint.TARGET_PARALLELISM, ReadHint.BATCH_SIZE} - ), - install_hint=_DAFT_CLICKHOUSE_INSTALL_HINT, - ) - - def _ray_clickhouse_descriptor() -> BindingDescriptor: from tributo.data.bindings.ray_clickhouse import RayClickHouseBinding @@ -673,20 +646,6 @@ def default_engine_bindings() -> EngineBindings: None, ("pylance", "daft-lance"), ), - ( - _daft_clickhouse_descriptor, - BindingKey( - "tributo.daft", - ScanKind.SQL, - "clickhouse", - "daft_clickhouse.daft.clickhouse", - ), - "daft", - _DAFT_SQL_VERSION_SPEC, - _DAFT_CLICKHOUSE_INSTALL_HINT, - None, - ("daft-clickhouse", "clickhouse-connect"), - ), ( _ray_clickhouse_descriptor, BindingKey( diff --git a/src/tributo/data/bindings/_daft_sql.py b/src/tributo/data/bindings/_daft_sql.py index 74b21ad0..d82bc1d2 100644 --- a/src/tributo/data/bindings/_daft_sql.py +++ b/src/tributo/data/bindings/_daft_sql.py @@ -1,4 +1,4 @@ -"""Shared Daft SQL bindings for independent database connector packages.""" +"""Daft SQL binding for the independent daft-doris connector package.""" from __future__ import annotations @@ -75,15 +75,14 @@ def compile(self, request: BindingCompileRequest) -> BindingCompilation: "target_parallelism" ), ) - if self.connector_id == "doris": - protocol = str(request.runtime_options.get("protocol") or "mysql") - if protocol not in {"mysql", "flight"}: - raise BindingStageError.framework_diagnostic( - "validate_capabilities", - error_type=JobConfigurationError, - diagnostic_code="unsupported_doris_transport", - diagnostic=("Doris transport must be 'mysql' or 'flight'"), - ) + protocol = str(request.runtime_options.get("protocol") or "mysql") + if protocol not in {"mysql", "flight"}: + raise BindingStageError.framework_diagnostic( + "validate_capabilities", + error_type=JobConfigurationError, + diagnostic_code="unsupported_doris_transport", + diagnostic=("Doris transport must be 'mysql' or 'flight'"), + ) with binding_stage("classify_transforms"): decisions = residual_decisions(request.transforms) with binding_stage("build_native_plan"): @@ -114,26 +113,19 @@ def _build( if target_tasks is not None: options["target_tasks"] = target_tasks - if self.connector_id == "clickhouse": - from daft_clickhouse import read_clickhouse + from daft_doris import read_doris - options["port"] = target.port - dataframe = read_clickhouse(**options) - transport_id = "clickhouse_native" - else: - from daft_doris import read_doris - - protocol = str(request.runtime_options.get("protocol") or "mysql") - options["transport"] = protocol - options["mysql_port"] = target.port - http_port = request.runtime_options.get("http_port") - flight_port = request.runtime_options.get("flight_port") - if http_port is not None: - options["http_port"] = int(http_port) - if flight_port is not None: - options["flight_port"] = int(flight_port) - dataframe = read_doris(**options) - transport_id = protocol + protocol = str(request.runtime_options.get("protocol") or "mysql") + options["transport"] = protocol + options["mysql_port"] = target.port + http_port = request.runtime_options.get("http_port") + flight_port = request.runtime_options.get("flight_port") + if http_port is not None: + options["http_port"] = int(http_port) + if flight_port is not None: + options["flight_port"] = int(flight_port) + dataframe = read_doris(**options) + transport_id = protocol schema = canonical_engine_schema(dataframe.schema()) transforms = ConcreteTransformCompiler().compile( @@ -178,13 +170,6 @@ def _wrap( ) -@PublicAPI(stability=Stability.ALPHA) -class DaftClickHouseBinding(_DaftSqlBinding): - connector_id = "clickhouse" - package_name = "daft-clickhouse" - reader_api = "daft_clickhouse.read_clickhouse" - - @PublicAPI(stability=Stability.ALPHA) class DaftDorisBinding(_DaftSqlBinding): connector_id = "doris" diff --git a/src/tributo/data/bindings/daft_clickhouse.py b/src/tributo/data/bindings/daft_clickhouse.py deleted file mode 100644 index 19c87df1..00000000 --- a/src/tributo/data/bindings/daft_clickhouse.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Daft ClickHouse Binding using the public daft-clickhouse facade.""" - -from tributo.data.bindings._daft_sql import DaftClickHouseBinding - -__all__ = ["DaftClickHouseBinding"] diff --git a/tests/data/test_engine_binding.py b/tests/data/test_engine_binding.py index 56397b48..d171b688 100644 --- a/tests/data/test_engine_binding.py +++ b/tests/data/test_engine_binding.py @@ -52,9 +52,16 @@ def installed_versions(monkeypatch: pytest.MonkeyPatch) -> None: "test-helper": "4.5.6", "tributo": "1.0.0", } + real_version = importlib.metadata.version + + def version_for_test(name: str) -> str: + if name in versions: + return versions[name] + return real_version(name) + monkeypatch.setattr( "tributo.data.engine_binding.importlib.metadata.version", - lambda name: versions[name], + version_for_test, ) @@ -733,19 +740,14 @@ def test_external_sql_descriptors_use_package_identity( builtin_bindings, "_distribution_version", lambda name: { - "daft-clickhouse": "1.0", "daft-doris": "1.0", "ray-clickhouse": "0.1.0", }.get(name), ) - clickhouse = builtin_bindings._daft_clickhouse_descriptor() doris = builtin_bindings._daft_doris_descriptor() ray_clickhouse = builtin_bindings._ray_clickhouse_descriptor() - assert clickhouse.key.binding_id == "daft_clickhouse.daft.clickhouse" - assert clickhouse.distribution_name == "daft-clickhouse" - assert clickhouse.dependency_distributions == ("clickhouse-connect",) assert doris.key.binding_id == "daft_doris.daft.doris" assert doris.distribution_name == "daft-doris" assert doris.dependency_distributions == ("PyMySQL",) @@ -778,7 +780,10 @@ def test_missing_external_sql_packages_report_install_hints( with pytest.raises( EngineNotAvailableError, - match=r"daft_clickhouse\.daft\.clickhouse.*daft-clickhouse.*tributo\[clickhouse\]", + match=( + r"No installed binding for " + r"tributo\.daft/sql/clickhouse/daft_clickhouse\.daft\.clickhouse" + ), ): bindings.resolve( BindingKey( diff --git a/tests/data/test_source_bindings.py b/tests/data/test_source_bindings.py index 52576aa5..8cd3831e 100644 --- a/tests/data/test_source_bindings.py +++ b/tests/data/test_source_bindings.py @@ -12,7 +12,6 @@ from tributo.data.bindings._postgresql import compile_table_query from tributo.data.bindings._sql_shared import resolve_sql_target -from tributo.data.bindings.daft_clickhouse import DaftClickHouseBinding from tributo.data.bindings.daft_csv import DaftCsvBinding from tributo.data.bindings.daft_doris import DaftDorisBinding from tributo.data.bindings.daft_iceberg import DaftIcebergBinding @@ -437,69 +436,12 @@ def test_lance_bindings_reject_iceberg_snapshot_refs( assert exc_info.value.diagnostic_code == "unsupported_lance_snapshot_ref" -def test_daft_sql_default_auto_sharding_is_delegated( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[dict[str, Any]] = [] - module = ModuleType("daft_clickhouse") - module.read_clickhouse = lambda **kwargs: calls.append(kwargs) or _DaftDataFrame() - monkeypatch.setitem(sys.modules, "daft_clickhouse", module) - monkeypatch.setattr( - "tributo.data.bindings._daft_sql.importlib.metadata.version", - lambda name: "0.7.23", - ) - plan = SqlScan( - provider_id="tributo.clickhouse", - connector_id="clickhouse", - target=SqlTableRead(schema="analytics", table="events"), - sharding=SqlShardRequirement(mode=SqlShardMode.AUTO), - ) - - DaftClickHouseBinding().compile( - _request( - plan, - runtime_options={ - "host": "db.example", - "port": 8123, - "database": "analytics", - }, - ) - ) - - assert calls[0]["split"] == "auto" - - -def test_daft_sql_single_read_rejects_parallelism_with_actionable_modes() -> None: - plan = SqlScan( - provider_id="tributo.clickhouse", - connector_id="clickhouse", - target=SqlTableRead(schema="analytics", table="events"), - ) - - with pytest.raises( - BindingStageError, - match=("partitioning.mode to 'auto'.*remove target_parallelism"), - ) as exc_info: - DaftClickHouseBinding().compile( - _request(plan, read_options=ReadOptions(target_parallelism=4)) - ) - - assert exc_info.value.diagnostic_code == "single_sql_read_rejects_parallelism_hint" - - -@pytest.mark.parametrize( - ("binding", "connector_id"), - [(DaftClickHouseBinding(), "clickhouse"), (DaftDorisBinding(), "doris")], -) -def test_daft_sql_parallel_reads_fail_closed( - binding: EngineBinding, - connector_id: str, -) -> None: +def test_daft_sql_parallel_reads_fail_closed() -> None: with pytest.raises( BindingStageError, match="explicit 'parallel'.*unsupported", ) as exc_info: - binding.compile(_request(_sql_plan(connector_id))) + DaftDorisBinding().compile(_request(_sql_plan("doris"))) assert exc_info.value.diagnostic_code == "parallel_sql_read_unsupported" @@ -531,7 +473,7 @@ def _sql_plan(connector_id: str) -> SqlScan: ) -def _daft_auto_sql_plan(connector_id: str) -> SqlScan: +def _auto_sql_plan(connector_id: str) -> SqlScan: return SqlScan( provider_id=f"tributo.{connector_id}", connector_id=connector_id, @@ -546,13 +488,6 @@ def _daft_auto_sql_plan(connector_id: str) -> SqlScan: @pytest.mark.parametrize( ("binding", "connector_id", "reader_name", "transport_id", "protocol"), [ - ( - DaftClickHouseBinding(), - "clickhouse", - "read_clickhouse", - "clickhouse_native", - "mysql", - ), (DaftDorisBinding(), "doris", "read_doris", "mysql", "mysql"), (DaftDorisBinding(), "doris", "read_doris", "flight", "flight"), ], @@ -582,10 +517,10 @@ def reader(**kwargs: Any) -> _DaftDataFrame: result = binding.compile( _request( - _daft_auto_sql_plan(connector_id), + _auto_sql_plan(connector_id), runtime_options={ "host": "db.example", - "port": 8123 if connector_id == "clickhouse" else 9030, + "port": 9030, "database": "analytics", "user": "reader", "password": "secret", @@ -600,11 +535,8 @@ def reader(**kwargs: Any) -> _DaftDataFrame: assert calls[0]["table"] == "events" assert calls[0]["target_tasks"] == 6 assert calls[0]["batch_rows"] == 128 - if connector_id == "clickhouse": - assert calls[0]["port"] == 8123 - else: - assert calls[0]["mysql_port"] == 9030 - assert calls[0]["transport"] == protocol + assert calls[0]["mysql_port"] == 9030 + assert calls[0]["transport"] == protocol assert result.reader_api == f"{module_name}.{reader_name}" assert result.transport_id == transport_id @@ -728,7 +660,7 @@ def test_ray_clickhouse_binding_maps_auto_to_partition( RayClickHouseBinding().compile( _request( - _daft_auto_sql_plan("clickhouse"), + _auto_sql_plan("clickhouse"), runtime_options={ "host": "clickhouse.example", "port": 8123, diff --git a/tests/integration/test_it_component_versions.py b/tests/integration/test_it_component_versions.py index 0bc263fd..c70dc0fa 100644 --- a/tests/integration/test_it_component_versions.py +++ b/tests/integration/test_it_component_versions.py @@ -124,7 +124,6 @@ def test_full_runtime_profile_matches_the_image_builder_contract() -> None: assert profile["version_contract"]["ray"] == "2.55.1" assert project["project"]["optional-dependencies"]["hive-ray"] == ["ray-hive==1.0"] for package, contract_key in { - "daft-clickhouse": "daft_clickhouse", "daft-doris": "daft_doris", "ray-doris": "ray_doris", "ray-hive": "ray_hive", diff --git a/tests/integrations/Dockerfile.data-ingestion b/tests/integrations/Dockerfile.data-ingestion index a687434e..b02187cd 100644 --- a/tests/integrations/Dockerfile.data-ingestion +++ b/tests/integrations/Dockerfile.data-ingestion @@ -20,6 +20,6 @@ WORKDIR /opt/tributo RUN --mount=type=cache,target=/home/ray/.cache/pip,uid=1000,gid=100 \ python -m pip install --require-hashes -r /opt/tributo/requirements.txt -RUN python -c "import importlib.metadata as m, sys; assert sys.version_info[:2] == (3, 12); assert m.version('pip'); assert m.version('ray') == '2.55.1'; assert m.version('daft').startswith('0.7.'); assert m.version('pylance') == '9.0.0'; assert m.version('lance-ray') == '0.5.0'; assert m.version('pyarrow') == '19.0.1'; assert m.version('daft-clickhouse') == '1.0'; assert m.version('daft-doris') == '1.0'; assert m.version('ray-doris') == '1.0'; assert m.version('ray-hive') == '1.0'; assert m.version('thrift') == '0.16.0'" +RUN python -c "import importlib.metadata as m, sys; assert sys.version_info[:2] == (3, 12); assert m.version('pip'); assert m.version('ray') == '2.55.1'; assert m.version('daft').startswith('0.7.'); assert m.version('pylance') == '9.0.0'; assert m.version('lance-ray') == '0.5.0'; assert m.version('pyarrow') == '19.0.1'; assert m.version('daft-doris') == '1.0'; assert m.version('ray-doris') == '1.0'; assert m.version('ray-hive') == '1.0'; assert m.version('thrift') == '0.16.0'" WORKDIR /workspace/tributo-src diff --git a/tests/integrations/README.md b/tests/integrations/README.md index b62ba4dd..4e332b44 100644 --- a/tests/integrations/README.md +++ b/tests/integrations/README.md @@ -7,8 +7,8 @@ pre-existing container. `ci/test-suites.json` owns their impact rules and reports required evidence, but no GitHub Actions event executes these suites. The data-ingestion image installs database drivers and the v1.0 database -connectors through Tributo extras. `daft-clickhouse==1.0`, -`daft-doris==1.0`, `ray-doris==1.0`, and `ray-hive==1.0` are resolved by +connectors through Tributo extras. `daft-doris==1.0`, `ray-doris==1.0`, +and `ray-hive==1.0` are resolved by `uv.lock`. The Data Ingestion Gate validates the Tributo Ray HiveServer2 Provider/Binding separately from the canonical full-runtime package-presence gate. The Ray ClickHouse Binding requires the external @@ -26,7 +26,6 @@ that package outside the Tributo lockfile. | S3/MinIO contract | `../integration/test_export_s3.py`, `../integration/test_minio_compat.py` | `ci_fast` Moto / `manual_external` MinIO | Manifest-last publication, Lease/CAS, alias, GC, path-style access, and conditional writes | Ephemeral Moto in CI; run-owned MinIO externally | | Runtime image gate | `../../scripts/run_runtime_image_it.sh`, `jobs/runtime_image_gate_job.py` | `manual_external` | Pinned full image, first-party and Alpha imports on driver/worker, Ray Data, Ray Jobs, and image attestations | Docker Buildx + two-node Ray cluster on a matching native host architecture | | MLflow Hook | `test_e2e_mlflow.py` | `manual_external` | Committed Bundle upload, replay deduplication, explicit run reuse, and failure semantics | Isolated model-export runner | -| ClickHouse E2E | `test_e2e_clickhouse.py` | `quarantine` | ClickHouse table → Daft ClickHouse Binding → explicit Daft-to-Ray adapter → XGBoost distributed training → MLflow → ONNX | Full image or `tributo[clickhouse]`; lifecycle and ownership contract pending | | Data Ingestion Docker | `test_data_ingestion_dual_engine.py` | `manual_external` | Local/S3 Parquet, Iceberg-on-MinIO Ray/Daft reads, native writes, Ray HiveServer2 structured projection, typed handles, and worker-version evidence | Docker Ray cluster + Daft + MinIO + Hive 4.2.0 | | Lance vector index | `test_lance_vector_index.py` | `manual_external` | Distributed IVF_FLAT/IVF_PQ build, append coverage, global Top-K, fallback, optimization, compaction, Ray Jobs, and S3 result delivery | Docker Ray cluster + Lance-Ray + MinIO | | File conformance | `../integration/test_data_ingestion_conformance.py` | `manual_external` | Local/MinIO Parquet and CSV through Ray Data and Daft | Local Ray runtime + MinIO | diff --git a/tests/integrations/jobs/runtime_image_gate_job.py b/tests/integrations/jobs/runtime_image_gate_job.py index ae00255d..2494b176 100644 --- a/tests/integrations/jobs/runtime_image_gate_job.py +++ b/tests/integrations/jobs/runtime_image_gate_job.py @@ -33,7 +33,6 @@ def worker_probe() -> dict[str, object]: "ray": importlib.metadata.version("ray"), "tributo": importlib.metadata.version("tributo"), "ray-hive": importlib.metadata.version("ray-hive"), - "daft-clickhouse": importlib.metadata.version("daft-clickhouse"), "daft-doris": importlib.metadata.version("daft-doris"), "ray-doris": importlib.metadata.version("ray-doris"), "thrift": importlib.metadata.version("thrift"), @@ -65,7 +64,6 @@ def main() -> None: "ray": "2.55.1", "tributo": "1.0.0", "ray-hive": "1.0", - "daft-clickhouse": "1.0", "daft-doris": "1.0", "ray-doris": "1.0", "thrift": "0.16.0", diff --git a/tests/integrations/runtime-profiles.json b/tests/integrations/runtime-profiles.json index 29e56fcb..894f2420 100644 --- a/tests/integrations/runtime-profiles.json +++ b/tests/integrations/runtime-profiles.json @@ -26,7 +26,6 @@ "uv_version": "0.11.23", "version_contract": { "daft_prefix": "0.7.", - "daft_clickhouse": "1.0", "daft_doris": "1.0", "lance_ray": "0.5.0", "pyarrow": "19.0.1", @@ -95,7 +94,6 @@ "uv_version": "0.11.23", "version_contract": { "daft_prefix": "0.7.", - "daft_clickhouse": "1.0", "daft_doris": "1.0", "lance_ray": "0.5.0", "pyarrow": "19.0.1", diff --git a/tests/vector_index/test_ci_contract.py b/tests/vector_index/test_ci_contract.py index 02428e3d..84670d36 100644 --- a/tests/vector_index/test_ci_contract.py +++ b/tests/vector_index/test_ci_contract.py @@ -69,7 +69,6 @@ def test_vector_runtime_is_exactly_versioned_in_the_shared_profile() -> None: assert "vector-index" in profile["extras"] assert profile["version_contract"] == { "daft_prefix": "0.7.", - "daft_clickhouse": "1.0", "daft_doris": "1.0", "lance_ray": "0.5.0", "pyarrow": "19.0.1", @@ -85,7 +84,6 @@ def test_vector_runtime_is_exactly_versioned_in_the_shared_profile() -> None: assert "m.version('pylance') == '9.0.0'" in dockerfile assert "m.version('lance-ray') == '0.5.0'" in dockerfile assert "m.version('pyarrow') == '19.0.1'" in dockerfile - assert "m.version('daft-clickhouse') == '1.0'" in dockerfile assert "m.version('daft-doris') == '1.0'" in dockerfile assert "m.version('ray-doris') == '1.0'" in dockerfile assert "m.version('ray-hive') == '1.0'" in dockerfile diff --git a/tools/runtime_image_contract.py b/tools/runtime_image_contract.py index d3b0efa8..4f6a3ae2 100644 --- a/tools/runtime_image_contract.py +++ b/tools/runtime_image_contract.py @@ -26,7 +26,6 @@ "mlflow", "grpc", "torch_geometric", - "daft_clickhouse", "daft_doris", "ray_doris", ) @@ -51,7 +50,6 @@ "dowhy", "econml", "bayesian-optimization", - "daft-clickhouse", "daft-doris", "ray-doris", "clickhouse-connect", @@ -63,7 +61,6 @@ REQUIRED_DISTRIBUTION_VERSIONS = { "ray-hive": "1.0", - "daft-clickhouse": "1.0", "daft-doris": "1.0", "ray-doris": "1.0", "thrift": "0.16.0", diff --git a/tools/tributo_it.py b/tools/tributo_it.py index 962a354b..2d970a4a 100644 --- a/tools/tributo_it.py +++ b/tools/tributo_it.py @@ -582,7 +582,6 @@ def _version_check_code(identity: RuntimeIdentity) -> str: ("lance_ray", "lance-ray"), ("pyarrow", "pyarrow"), ("ray_hive", "ray-hive"), - ("daft_clickhouse", "daft-clickhouse"), ("daft_doris", "daft-doris"), ("ray_doris", "ray-doris"), ("thrift", "thrift"), diff --git a/uv.lock b/uv.lock index fc7ed4d8..d874b34a 100644 --- a/uv.lock +++ b/uv.lock @@ -880,24 +880,6 @@ ray = [ { name = "ray", extra = ["client", "data"] }, ] -[[package]] -name = "daft-clickhouse" -version = "1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "daft" }, - { name = "pyarrow" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b6/0b/4857e4746fdbe8f1cd911854e4e4821d1650b7f2c55efb558fcbad2707aa/daft_clickhouse-1.0.tar.gz", hash = "sha256:053ee0b740eb68246b48c3cf7bcce8eb8e9766e6c9530968a80f3b0fa08ce77a", size = 169506, upload-time = "2026-08-17T14:44:58.758Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/b2/c32e73be7521cbd0e6eb5fbbe00114a5dcb637265a7db1e5f7a94bf6af15/daft_clickhouse-1.0-py3-none-any.whl", hash = "sha256:6b3ce6d2dc30fb0c282cfa99f14faa41887e139f31424f16ebc8b4d090ee1c7f", size = 51936, upload-time = "2026-08-17T14:44:57.553Z" }, -] - -[package.optional-dependencies] -clickhouse = [ - { name = "clickhouse-connect", extra = ["arrow", "async"] }, -] - [[package]] name = "daft-doris" version = "1.0" @@ -4814,7 +4796,6 @@ causal = [ ] clickhouse = [ { name = "clickhouse-connect", extra = ["arrow", "async"] }, - { name = "daft-clickhouse", extra = ["clickhouse"] }, ] data = [ { name = "lance-ray" }, @@ -4943,7 +4924,6 @@ test-integration = [ { name = "causal-learn" }, { name = "clickhouse-connect", extra = ["arrow", "async"] }, { name = "daft", extra = ["lance", "ray"] }, - { name = "daft-clickhouse", extra = ["clickhouse"] }, { name = "daft-doris", extra = ["doris"] }, { name = "dowhy" }, { name = "lance-ray" }, @@ -4971,7 +4951,6 @@ training = [ { name = "boto3" }, { name = "clickhouse-connect", extra = ["arrow", "async"] }, { name = "daft", extra = ["lance", "ray"] }, - { name = "daft-clickhouse", extra = ["clickhouse"] }, { name = "daft-doris", extra = ["doris"] }, { name = "lance-ray" }, { name = "ml-dtypes" }, @@ -5029,7 +5008,6 @@ requires-dist = [ { name = "click", specifier = ">=8.0.0" }, { name = "clickhouse-connect", extras = ["arrow", "async"], marker = "extra == 'clickhouse'", specifier = ">=1.5,<1.6" }, { name = "daft", extras = ["lance", "ray"], marker = "extra == 'data-daft'", specifier = ">=0.7.23,<0.7.24" }, - { name = "daft-clickhouse", extras = ["clickhouse"], marker = "extra == 'clickhouse'", specifier = "==1.0" }, { name = "daft-doris", extras = ["doris"], marker = "extra == 'mysql'", specifier = "==1.0" }, { name = "daft-doris", extras = ["doris-flight"], marker = "extra == 'doris-flight'", specifier = "==1.0" }, { name = "dowhy", marker = "extra == 'causal'", specifier = ">=0.11.0" },