diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 2c8ecbc5e..841bf205a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -16,7 +16,12 @@ # under the License. name: Dev -on: [push, pull_request] +on: + push: + branches: + - main + - branch-* + pull_request: jobs: diff --git a/Cargo.lock b/Cargo.lock index fc9b74cdb..a6dad3592 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,8 +791,6 @@ dependencies = [ [[package]] name = "datafusion" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" dependencies = [ "arrow", "arrow-schema", @@ -845,8 +843,6 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" dependencies = [ "arrow", "async-trait", @@ -870,8 +866,6 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" dependencies = [ "arrow", "async-trait", @@ -893,8 +887,6 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" dependencies = [ "arrow", "arrow-ipc", @@ -919,8 +911,6 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" dependencies = [ "futures", "log", @@ -930,8 +920,6 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" dependencies = [ "arrow", "async-compression", @@ -966,8 +954,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" dependencies = [ "arrow", "arrow-ipc", @@ -990,8 +976,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" dependencies = [ "arrow", "arrow-avro", @@ -1009,8 +993,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" dependencies = [ "arrow", "async-trait", @@ -1032,8 +1014,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" dependencies = [ "arrow", "async-trait", @@ -1055,8 +1035,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" dependencies = [ "arrow", "async-trait", @@ -1086,14 +1064,10 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" [[package]] name = "datafusion-execution" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" dependencies = [ "arrow", "arrow-buffer", @@ -1114,8 +1088,6 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" dependencies = [ "arrow", "arrow-schema", @@ -1137,8 +1109,6 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" dependencies = [ "arrow", "datafusion-common", @@ -1149,8 +1119,6 @@ dependencies = [ [[package]] name = "datafusion-ffi" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152" dependencies = [ "arrow", "arrow-schema", @@ -1181,7 +1149,7 @@ dependencies = [ [[package]] name = "datafusion-ffi-example" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow", "arrow-array", @@ -1204,8 +1172,6 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" dependencies = [ "arrow", "arrow-buffer", @@ -1236,8 +1202,6 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" dependencies = [ "arrow", "datafusion-common", @@ -1257,8 +1221,6 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" dependencies = [ "arrow", "datafusion-common", @@ -1269,8 +1231,6 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" dependencies = [ "arrow", "arrow-ord", @@ -1294,8 +1254,6 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" dependencies = [ "arrow", "async-trait", @@ -1310,8 +1268,6 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" dependencies = [ "arrow", "datafusion-common", @@ -1327,8 +1283,6 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1337,8 +1291,6 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" dependencies = [ "datafusion-doc", "quote", @@ -1348,8 +1300,6 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" dependencies = [ "arrow", "chrono", @@ -1368,8 +1318,6 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" dependencies = [ "arrow", "datafusion-common", @@ -1390,8 +1338,6 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" dependencies = [ "arrow", "datafusion-common", @@ -1405,8 +1351,6 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" dependencies = [ "arrow", "chrono", @@ -1422,8 +1366,6 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" dependencies = [ "arrow", "datafusion-common", @@ -1434,6 +1376,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", + "datafusion-session", "itertools", "recursive", ] @@ -1441,8 +1384,6 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" dependencies = [ "arrow", "arrow-data", @@ -1474,8 +1415,6 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" dependencies = [ "arrow", "chrono", @@ -1501,8 +1440,6 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" dependencies = [ "arrow", "datafusion-common", @@ -1512,8 +1449,6 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" dependencies = [ "arrow", "datafusion-common", @@ -1527,7 +1462,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow", "arrow-select", @@ -1559,7 +1494,7 @@ dependencies = [ [[package]] name = "datafusion-python-util" -version = "53.0.0" +version = "54.0.0" dependencies = [ "arrow", "datafusion", @@ -1573,8 +1508,6 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" dependencies = [ "async-trait", "datafusion-common", @@ -1587,13 +1520,12 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", + "datafusion", "datafusion-catalog", "datafusion-common", "datafusion-execution", @@ -1616,8 +1548,6 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" dependencies = [ "arrow", "bigdecimal", @@ -1635,8 +1565,6 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 0f3236ecf..d1e362634 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [workspace.package] -version = "53.0.0" +version = "54.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] @@ -40,17 +40,17 @@ arrow = { version = "58" } arrow-array = { version = "58" } arrow-schema = { version = "58" } arrow-select = { version = "58" } -datafusion = { version = "54" } -datafusion-substrait = { version = "54" } -datafusion-proto = { version = "54" } -datafusion-ffi = { version = "54" } -datafusion-catalog = { version = "54", default-features = false } -datafusion-common = { version = "54", default-features = false } -datafusion-functions-aggregate = { version = "54" } -datafusion-functions-window = { version = "54" } -datafusion-spark = { version = "54" } -datafusion-expr = { version = "54" } -prost = "0.14.3" +datafusion = { path = "../datafusion/datafusion/core", version = "54" } +datafusion-substrait = { path = "../datafusion/datafusion/substrait", version = "54" } +datafusion-proto = { path = "../datafusion/datafusion/proto", version = "54" } +datafusion-ffi = { path = "../datafusion/datafusion/ffi", version = "54" } +datafusion-catalog = { path = "../datafusion/datafusion/catalog", version = "54", default-features = false } +datafusion-common = { path = "../datafusion/datafusion/common", version = "54", default-features = false } +datafusion-functions-aggregate = { path = "../datafusion/datafusion/functions-aggregate", version = "54" } +datafusion-functions-window = { path = "../datafusion/datafusion/functions-window", version = "54" } +datafusion-spark = { path = "../datafusion/datafusion/spark", version = "54", features = ["datafusion"] } +datafusion-expr = { path = "../datafusion/datafusion/expr", version = "54" } +prost = "0.14.1" serde_json = "1" uuid = { version = "1.23" } mimalloc = { version = "0.1", default-features = false } @@ -61,9 +61,9 @@ object_store = { version = "0.13.1" } url = "2" log = "0.4.29" parking_lot = "0.12" -prost-types = "0.14.3" # keep in line with `datafusion-substrait` +prost-types = "0.14.1" # keep in line with `datafusion-substrait` pyo3-build-config = "0.28" -datafusion-python-util = { path = "crates/util", version = "53.0.0" } +datafusion-python-util = { path = "crates/util", version = "54.0.0" } [profile.release] lto = "thin" diff --git a/crates/core/src/context.rs b/crates/core/src/context.rs index 0613a96dc..0e0c59375 100644 --- a/crates/core/src/context.rs +++ b/crates/core/src/context.rs @@ -24,6 +24,7 @@ use std::sync::Arc; use arrow::array::RecordBatchReader; use arrow::ffi_stream::ArrowArrayStreamReader; use arrow::pyarrow::FromPyArrow; +use async_trait::async_trait; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; @@ -36,14 +37,16 @@ use datafusion::datasource::listing::{ }; use datafusion::datasource::{MemTable, TableProvider}; use datafusion::execution::context::{ - DataFilePaths, SQLOptions, SessionConfig, SessionContext, TaskContext, + DataFilePaths, QueryPlanner, SQLOptions, SessionConfig, SessionContext, TaskContext, }; use datafusion::execution::disk_manager::DiskManagerMode; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; use datafusion::execution::options::{ArrowReadOptions, ReadOptions}; use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::execution::session_state::SessionStateBuilder; -use datafusion::execution::{FunctionRegistry, TaskContextProvider}; +use datafusion::execution::{FunctionRegistry, Session, TaskContextProvider}; +use datafusion::logical_expr::LogicalPlan; +use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, JsonReadOptions, ParquetReadOptions, }; @@ -53,6 +56,7 @@ use datafusion_ffi::config::extension_options::FFI_ExtensionOptions; use datafusion_ffi::execution::FFI_TaskContextProvider; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; +use datafusion_ffi::query_planner::FFI_QueryPlanner; use datafusion_ffi::table_provider_factory::FFI_TableProviderFactory; use datafusion_proto::logical_plan::LogicalExtensionCodec; use datafusion_proto::physical_plan::PhysicalExtensionCodec; @@ -221,6 +225,39 @@ impl PySessionConfig { } } +#[derive(Debug, Clone)] +struct PythonRuntimeQueryPlanner { + planner: FFI_QueryPlanner, +} + +#[async_trait] +impl QueryPlanner for PythonRuntimeQueryPlanner { + async fn create_physical_plan( + &self, + logical_plan: &LogicalPlan, + session: &dyn Session, + ) -> datafusion::common::Result> { + let runtime = get_tokio_runtime().handle().clone(); + self.planner + .create_physical_plan_with_session_runtime(logical_plan, session, Some(runtime)) + .await + } +} + +fn ffi_query_planner_from_pycapsule(obj: &Bound) -> PyResult { + let capsule = if obj.hasattr("__datafusion_query_planner__")? { + obj.getattr("__datafusion_query_planner__")?.call0()? + } else { + obj.clone() + }; + + let capsule = capsule.cast::()?; + let data: NonNull = capsule + .pointer_checked(Some(c"datafusion_query_planner_v1"))? + .cast(); + Ok(unsafe { data.as_ref().clone() }) +} + /// Runtime options for a SessionContext #[pyclass( from_py_object, @@ -1199,6 +1236,21 @@ impl PySessionContext { Ok(()) } + pub fn with_query_planner(&self, query_planner: Bound<'_, PyAny>) -> PyDataFusionResult { + let planner = ffi_query_planner_from_pycapsule(&query_planner)?; + let planner = Arc::new(PythonRuntimeQueryPlanner { planner }); + let state = SessionStateBuilder::new_from_existing(self.ctx.state()) + .with_query_planner(planner) + .build(); + let ctx = Arc::new(SessionContext::new_with_state(state)); + + Ok(Self { + ctx, + logical_codec: Arc::clone(&self.logical_codec), + physical_codec: Arc::clone(&self.physical_codec), + }) + } + pub fn table_provider(&self, name: &str, py: Python) -> PyResult { let provider = wait_for_future(py, self.ctx.table_provider(name)) // Outer error: runtime/async failure @@ -1375,6 +1427,19 @@ impl PySessionContext { create_logical_extension_capsule(py, ffi.as_ref()) } + pub fn __datafusion_query_planner__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let planner = Arc::clone(self.ctx.state().query_planner()); + let ffi = FFI_QueryPlanner::new_with_ffi_codecs( + planner, + self.ffi_logical_codec().as_ref().clone(), + self.ffi_physical_codec().as_ref().clone(), + ); + PyCapsule::new(py, ffi, Some(cr"datafusion_query_planner_v1".into())) + } + pub fn with_logical_extension_codec<'py>( &self, codec: Bound<'py, PyAny>, diff --git a/crates/core/src/expr/drop_catalog_schema.rs b/crates/core/src/expr/drop_catalog_schema.rs index fd5105332..f349098b7 100644 --- a/crates/core/src/expr/drop_catalog_schema.rs +++ b/crates/core/src/expr/drop_catalog_schema.rs @@ -18,9 +18,8 @@ use std::fmt::{self, Display, Formatter}; use std::sync::Arc; -use datafusion::common::SchemaReference; +use datafusion::common::{SchemaReference, TableReference}; use datafusion::logical_expr::DropCatalogSchema; -use datafusion::sql::TableReference; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; diff --git a/crates/util/src/lib.rs b/crates/util/src/lib.rs index 07aa0a2d5..0cc2d6e45 100644 --- a/crates/util/src/lib.rs +++ b/crates/util/src/lib.rs @@ -22,13 +22,14 @@ use std::time::Duration; use datafusion::datasource::TableProvider; use datafusion::execution::TaskContext; -use datafusion::execution::context::SessionContext; +use datafusion::execution::context::{QueryPlanner, SessionContext}; use datafusion::logical_expr::Volatility; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion_ffi::execution::FFI_TaskContextProvider; use datafusion_ffi::physical_optimizer::FFI_PhysicalOptimizerRule; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::proto::physical_extension_codec::FFI_PhysicalExtensionCodec; +use datafusion_ffi::query_planner::FFI_QueryPlanner; use datafusion_ffi::table_provider::FFI_TableProvider; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use pyo3::exceptions::{PyImportError, PyTypeError, PyValueError}; @@ -341,6 +342,13 @@ from_pycapsule!( dyn PhysicalOptimizerRule + Send + Sync ); +from_pycapsule!( + query_planner_from_pycapsule, + "datafusion_query_planner_v1", + FFI_QueryPlanner, + dyn QueryPlanner + Send + Sync +); + try_from_pycapsule!( task_context_from_pycapsule, "datafusion_task_context_provider", diff --git a/dev/changelog/54.0.0.md b/dev/changelog/54.0.0.md new file mode 100644 index 000000000..e73118d8b --- /dev/null +++ b/dev/changelog/54.0.0.md @@ -0,0 +1,106 @@ + + +# Apache DataFusion Python 54.0.0 Changelog + +This release consists of 51 commits from 8 contributors. See credits at the end of this changelog for more information. + +**Breaking changes:** + +- Update datafusion dependency to latest in preparation for DF54 [#1532](https://github.com/apache/datafusion-python/pull/1532) (timsaucer) +- feat: enable pickling for Python aggregate and window UDFs [#1545](https://github.com/apache/datafusion-python/pull/1545) (timsaucer) +- feat: pass calling SessionContext to Python UDTF callbacks [#1555](https://github.com/apache/datafusion-python/pull/1555) (timsaucer) +- feat: accept distinct kwarg on sum and avg [#1556](https://github.com/apache/datafusion-python/pull/1556) (timsaucer) + +**Implemented enhancements:** + +- feat: add AI skill to find and improve the Pythonic interface to functions [#1484](https://github.com/apache/datafusion-python/pull/1484) (timsaucer) +- feat: enable pickling of most Expr except udaf and udwf [#1544](https://github.com/apache/datafusion-python/pull/1544) (timsaucer) +- feat: expose variety of features from DF54 update [#1554](https://github.com/apache/datafusion-python/pull/1554) (timsaucer) +- feat: Python UDFs: per-session inlining toggle and strict refusal setting [#1546](https://github.com/apache/datafusion-python/pull/1546) (timsaucer) +- feat: create free-threaded python wheels [#1553](https://github.com/apache/datafusion-python/pull/1553) (timsaucer) +- feat: expose lambda and higher-order array functions [#1561](https://github.com/apache/datafusion-python/pull/1561) (timsaucer) +- feat: import user-defined physical optimizer rules over FFI [#1557](https://github.com/apache/datafusion-python/pull/1557) (timsaucer) +- feat: expose SessionContext.copied_config and parse_capacity_limit [#1570](https://github.com/apache/datafusion-python/pull/1570) (timsaucer) +- feat: expose array_compact, array_normalize, cosine_distance, inner_product [#1567](https://github.com/apache/datafusion-python/pull/1567) (timsaucer) +- feat: expose arrow_field, arrow_try_cast, cast_to_type, with_metadata [#1568](https://github.com/apache/datafusion-python/pull/1568) (timsaucer) +- feat: expose spark-compatible functions [#1564](https://github.com/apache/datafusion-python/pull/1564) (timsaucer) +- feat: improve pythonic interface on date/time functions [#1563](https://github.com/apache/datafusion-python/pull/1563) (timsaucer) + +**Fixed bugs:** + +- fix: type scalar UDF returns as Arrow arrays [#1528](https://github.com/apache/datafusion-python/pull/1528) (BharatDeva) +- fix: Skip `fork` and `forkserver` on `win32` [#1566](https://github.com/apache/datafusion-python/pull/1566) (nuno-faria) + +**Documentation updates:** + +- docs: enrich module docstrings and add doctest examples [#1498](https://github.com/apache/datafusion-python/pull/1498) (timsaucer) +- docs: add README section for AI coding assistants [#1503](https://github.com/apache/datafusion-python/pull/1503) (timsaucer) +- docs: add upstream sync process documentation [#1524](https://github.com/apache/datafusion-python/pull/1524) (timsaucer) +- docs: document null-handling function arguments [#1527](https://github.com/apache/datafusion-python/pull/1527) (BharatDeva) +- docs: user guide + runnable examples for distributing expressions [#1547](https://github.com/apache/datafusion-python/pull/1547) (timsaucer) +- docs: convert reStructuredText sources to MyST markdown [#1579](https://github.com/apache/datafusion-python/pull/1579) (timsaucer) + +**Other:** + +- Release 53.0.0 [#1491](https://github.com/apache/datafusion-python/pull/1491) (timsaucer) +- ci: disable symbol export on Windows verification [#1486](https://github.com/apache/datafusion-python/pull/1486) (timsaucer) +- Add Python bindings for accessing ExecutionMetrics [#1381](https://github.com/apache/datafusion-python/pull/1381) (ShreyeshArangath) +- Support None comparisons for null expressions [#1489](https://github.com/apache/datafusion-python/pull/1489) (zeel2104) +- chore: update release documentation [#1494](https://github.com/apache/datafusion-python/pull/1494) (timsaucer) +- Fix error on show() with an explain plan [#1492](https://github.com/apache/datafusion-python/pull/1492) (timsaucer) +- Add SKILL.md and enrich package docstring [#1497](https://github.com/apache/datafusion-python/pull/1497) (timsaucer) +- minor: fix header on agent SKILL.md file [#1501](https://github.com/apache/datafusion-python/pull/1501) (timsaucer) +- tpch examples: rewrite queries idiomatically and embed reference SQL [#1504](https://github.com/apache/datafusion-python/pull/1504) (timsaucer) +- Move public skills to a directory to avoid downloading the whole repo [#1519](https://github.com/apache/datafusion-python/pull/1519) (ntjohnson1) +- Update user documentation for AI agent skill usage [#1505](https://github.com/apache/datafusion-python/pull/1505) (timsaucer) +- build(deps): combined dependabot bumps (Cargo + workflows) [#1534](https://github.com/apache/datafusion-python/pull/1534) (timsaucer) +- Add support for logical and physical codecs [#1541](https://github.com/apache/datafusion-python/pull/1541) (timsaucer) +- Add details on caching to skill [#1521](https://github.com/apache/datafusion-python/pull/1521) (ntjohnson1) +- Bump DataFusion to prepare for DF54 release candidate [#1562](https://github.com/apache/datafusion-python/pull/1562) (timsaucer) +- Export `to_datafusion_err` from the util crate root [#1487](https://github.com/apache/datafusion-python/pull/1487) (kosiew) +- chore: remove unused PyConfig [#1485](https://github.com/apache/datafusion-python/pull/1485) (timsaucer) +- refactor(context): deduplicate register/read option-building logic [#1479](https://github.com/apache/datafusion-python/pull/1479) (mesejo) +- Improve documentation site layout [#1578](https://github.com/apache/datafusion-python/pull/1578) (timsaucer) +- Allow `DataFrame.aggregate` to accept `None` for no grouping [#1581](https://github.com/apache/datafusion-python/pull/1581) (kosiew) +- Update to released DF 54.0.0 [#1588](https://github.com/apache/datafusion-python/pull/1588) (timsaucer) +- build(deps): batch dependabot dependency updates [#1589](https://github.com/apache/datafusion-python/pull/1589) (timsaucer) +- Deprecate `Expr` temporal part arguments in date extraction and truncation functions [#1587](https://github.com/apache/datafusion-python/pull/1587) (kosiew) +- chore: update rust dependencies [#1604](https://github.com/apache/datafusion-python/pull/1604) (timsaucer) +- Add deprecation warnings for Expr passed to confirmed literal-only function arguments [#1605](https://github.com/apache/datafusion-python/pull/1605) (kosiew) +- chore: resolve audits after DF54.0.0 update to skill [#1608](https://github.com/apache/datafusion-python/pull/1608) (timsaucer) +- chore: remove 3.13 freethreaded builds [#1609](https://github.com/apache/datafusion-python/pull/1609) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 39 Tim Saucer + 4 kosiew + 2 BharatDeva + 2 Nick + 1 Daniel Mesejo + 1 Nuno Faria + 1 Shreyesh + 1 Zeel Desai +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index d93ff3c24..70f60168f 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -51,3 +51,5 @@ uv.lock examples/tpch/answers_sf1/*.tbl **/SKILL.md docs/source/llms.txt +CLAUDE.md +.claude/skills \ No newline at end of file diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 9591e0335..503e0de67 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -153,7 +153,7 @@ test_source_distribution() { #TODO: we should really run tests here as well #python3 -m pytest - if ( find -iname 'Cargo.toml' | xargs grep SNAPSHOT ); then + if ( find . -iname 'Cargo.toml' | xargs grep SNAPSHOT ); then echo "Cargo.toml version should not contain SNAPSHOT for releases" exit 1 fi diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 0bfc59bfe..49ae88256 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -143,6 +143,16 @@ class PhysicalOptimizerRuleExportable(Protocol): def __datafusion_physical_optimizer_rule__(self) -> object: ... # noqa: D105 +class QueryPlannerExportable(Protocol): + """Type hint for object that has __datafusion_query_planner__ PyCapsule. + + The method returns a PyCapsule wrapping an ``FFI_QueryPlanner``, typically + produced by a separate compiled extension. + """ + + def __datafusion_query_planner__(self) -> object: ... # noqa: D105 + + class SessionConfig: """Session configuration options.""" @@ -1657,6 +1667,25 @@ def add_physical_optimizer_rule( """ self.ctx.add_physical_optimizer_rule(rule) + def with_query_planner( + self, planner: QueryPlannerExportable | _PyCapsule + ) -> SessionContext: + """Create a new session context with a custom FFI query planner. + + The planner is imported via its ``__datafusion_query_planner__`` + PyCapsule, typically produced by a separate compiled extension. The + returned context shares this context's installed logical and physical + codecs and preserves the existing session state. + + Args: + planner: Object exposing ``__datafusion_query_planner__`` or a raw + ``datafusion_query_planner_v1`` PyCapsule. + """ + new_internal = self.ctx.with_query_planner(planner) + new = SessionContext.__new__(SessionContext) + new.ctx = new_internal + return new + def table_provider(self, name: str) -> Table: """Return the :py:class:`~datafusion.catalog.Table` for the given table name. @@ -2021,6 +2050,10 @@ def __datafusion_logical_extension_codec__(self) -> Any: """Access the PyCapsule FFI_LogicalExtensionCodec.""" return self.ctx.__datafusion_logical_extension_codec__() + def __datafusion_query_planner__(self) -> Any: + """Access the PyCapsule FFI_QueryPlanner for the current planner.""" + return self.ctx.__datafusion_query_planner__() + def with_logical_extension_codec( self, codec: LogicalExtensionCodecExportable | _PyCapsule ) -> SessionContext: diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 112a6fd7b..cd63f8f7e 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -714,6 +714,17 @@ def test_remove_optimizer_rule(ctx): assert ctx.remove_optimizer_rule("nonexistent_rule") is False +def test_with_query_planner_rejects_non_capsule(ctx): + with pytest.raises(TypeError, match="not an instance of 'PyCapsule'"): + ctx.with_query_planner(object()) + + +def test_with_query_planner_round_trip(ctx): + planner_context = ctx.with_query_planner(ctx) + batches = planner_context.sql("SELECT 1 AS value").collect() + assert batches[0].column(0) == pa.array([1]) + + def test_table_provider(ctx): batch = pa.RecordBatch.from_pydict({"x": [10, 20, 30]}) ctx.register_record_batches("provider_test", [[batch]]) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 894d81fcf..711c15ce0 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -145,11 +145,11 @@ def test_math_functions(): f.pow(col_v, literal(pa.scalar(4))), f.round(col_v), f.round(col_v, literal(pa.scalar(3))), - f.sqrt(col_v), + f.sqrt(f.abs(col_v)), f.signum(col_v), f.trunc(col_v), f.asinh(col_v), - f.acosh(col_v), + f.acosh(col_v + literal(pa.scalar(1))), f.atanh(col_v), f.cbrt(col_v), f.cosh(col_v), @@ -189,11 +189,11 @@ def test_math_functions(): np.testing.assert_array_almost_equal(result.column(16), np.power(values, 4)) np.testing.assert_array_almost_equal(result.column(17), np.round(values)) np.testing.assert_array_almost_equal(result.column(18), np.round(values, 3)) - np.testing.assert_array_almost_equal(result.column(19), np.sqrt(values)) + np.testing.assert_array_almost_equal(result.column(19), np.sqrt(np.abs(values))) np.testing.assert_array_almost_equal(result.column(20), np.sign(values)) np.testing.assert_array_almost_equal(result.column(21), np.trunc(values)) np.testing.assert_array_almost_equal(result.column(22), np.arcsinh(values)) - np.testing.assert_array_almost_equal(result.column(23), np.arccosh(values)) + np.testing.assert_array_almost_equal(result.column(23), np.arccosh(values + 1.0)) np.testing.assert_array_almost_equal(result.column(24), np.arctanh(values)) np.testing.assert_array_almost_equal(result.column(25), np.cbrt(values)) np.testing.assert_array_almost_equal(result.column(26), np.cosh(values))