From 0db3392c5b2ef47952b72428d6155c4fdf20f838 Mon Sep 17 00:00:00 2001 From: Peter Bower <37089506+pbower@users.noreply.github.com> Date: Mon, 21 Sep 2026 04:14:08 +0100 Subject: [PATCH 1/2] Add PyInput implementations --- minarrow-py/Cargo.lock | 10 +- minarrow-py/Cargo.toml | 5 + minarrow-py/examples/roundtrip_to_python.rs | 206 ++++++++++++++++++++ minarrow-py/src/pyliquid.rs | 162 ++++++++++++++- 4 files changed, 377 insertions(+), 6 deletions(-) create mode 100644 minarrow-py/examples/roundtrip_to_python.rs diff --git a/minarrow-py/Cargo.lock b/minarrow-py/Cargo.lock index f02a912..c0eb64c 100644 --- a/minarrow-py/Cargo.lock +++ b/minarrow-py/Cargo.lock @@ -28,7 +28,7 @@ checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "minarrow" -version = "0.18.1" +version = "0.18.2" dependencies = [ "log", "num-traits", @@ -37,7 +37,7 @@ dependencies = [ [[package]] name = "minarrow-py" -version = "0.18.1" +version = "0.18.2" dependencies = [ "minarrow", "minarrow-pyo3", @@ -48,7 +48,7 @@ dependencies = [ [[package]] name = "minarrow-pyo3" -version = "0.18.1" +version = "0.18.2" dependencies = [ "minarrow", "pyo3", @@ -197,6 +197,6 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "vec64" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef6bbef159f21ac387220ebc71141524423f7886afd390bc7b140f12630425" +checksum = "54db55dbf75205d7a52f11a60aa6742d0ce6a9eb3ed97346556f26aec246f139" diff --git a/minarrow-py/Cargo.toml b/minarrow-py/Cargo.toml index 0f101f4..f0bba83 100644 --- a/minarrow-py/Cargo.toml +++ b/minarrow-py/Cargo.toml @@ -96,6 +96,11 @@ name = "roundtrip_tensor" path = "examples/roundtrip_tensor.rs" required-features = ["embed", "ndarray"] +[[example]] +name = "roundtrip_to_python" +path = "examples/roundtrip_to_python.rs" +required-features = ["embed"] + [build-dependencies] pyo3-build-config = { version = "0.29", features = ["resolve-config"] } diff --git a/minarrow-py/examples/roundtrip_to_python.rs b/minarrow-py/examples/roundtrip_to_python.rs new file mode 100644 index 0000000..b083349 --- /dev/null +++ b/minarrow-py/examples/roundtrip_to_python.rs @@ -0,0 +1,206 @@ +// Copyright 2025 Peter Garfield Bower +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Roundtrip tests for every `PyInput` implementation. +//! +//! Each test sends a Minarrow value to Python through `to_python`, then +//! returns it immediately. `classify` on the Python side maps the result +//! back to a Minarrow `Value`, and the test verifies the data survived +//! the round trip. +//! +//! ## Running +//! ```bash +//! cd minarrow-py +//! PYO3_PYTHON=$PWD/../pyo3/.venv/bin/python \ +//! PYTHONHOME=/usr \ +//! PYTHONPATH=$PWD/../pyo3/.venv/lib/python3.12/site-packages \ +//! LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu \ +//! cargo run --example roundtrip_to_python --features embed +//! ``` + +use std::sync::Arc; + +use minarrow::{ + arr_f64, fa_f64, fa_str32, Array, ArrayV, FieldArray, Scalar, SuperArray, + SuperTable, Table, TableV, Value, +}; +use minarrow_py::{PyInput, PyLiquid}; +use pyo3::prelude::*; + +fn main() -> PyResult<()> { + let rt = PyLiquid::start(); + + roundtrip_table(&rt); + roundtrip_array(&rt); + roundtrip_field_array(&rt); + roundtrip_table_view(&rt); + roundtrip_array_view(&rt); + roundtrip_super_table(&rt); + roundtrip_super_array(&rt); + roundtrip_scalar_float(&rt); + roundtrip_scalar_int(&rt); + roundtrip_scalar_bool(&rt); + roundtrip_scalar_string(&rt); + roundtrip_scalar_null(&rt); + roundtrip_value_table(&rt); + roundtrip_value_array(&rt); + roundtrip_value_scalar(&rt); + + println!("All PyInput roundtrip tests passed."); + Ok(()) +} + +fn roundtrip_table(rt: &PyLiquid) { + let table = Table::new("test", Some(vec![ + fa_f64!("x", 1.0, 2.0, 3.0), + fa_str32!("y", "a", "b", "c"), + ])); + let result = rt.with_python(&table, |_py, obj| Ok(obj)).unwrap(); + let Value::Table(t) = result else { panic!("expected Table, got {result:?}") }; + assert_eq!(t.n_rows, 3); + assert_eq!(t.n_cols(), 2); + println!(" Table roundtrip: ok"); +} + +fn roundtrip_array(rt: &PyLiquid) { + let array: Array = arr_f64![10.0, 20.0, 30.0].into(); + let result = rt.with_python(&array, |_py, obj| Ok(obj)).unwrap(); + let Value::Array(a) = result else { panic!("expected Array, got {result:?}") }; + assert_eq!(a.len(), 3); + println!(" Array roundtrip: ok"); +} + +fn roundtrip_field_array(rt: &PyLiquid) { + let fa = fa_f64!("amount", 1.5, 2.5); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| fa.to_python(py)).unwrap(); + let Value::Array(a) = result else { panic!("expected Array, got {result:?}") }; + assert_eq!(a.len(), 2); + println!(" FieldArray roundtrip: ok"); +} + +fn roundtrip_table_view(rt: &PyLiquid) { + let table = Table::new("test", Some(vec![fa_f64!("x", 1.0, 2.0, 3.0, 4.0)])); + let view = TableV::from(table); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| view.to_python(py)).unwrap(); + let Value::Table(t) = result else { panic!("expected Table, got {result:?}") }; + assert_eq!(t.n_rows, 4); + println!(" TableV roundtrip: ok"); +} + +fn roundtrip_array_view(rt: &PyLiquid) { + let array: Array = arr_f64![5.0, 6.0, 7.0].into(); + let view = ArrayV::from(array); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| view.to_python(py)).unwrap(); + let Value::Array(a) = result else { panic!("expected Array, got {result:?}") }; + assert_eq!(a.len(), 3); + println!(" ArrayV roundtrip: ok"); +} + +fn roundtrip_super_table(rt: &PyLiquid) { + let t1 = Table::new("t", Some(vec![fa_f64!("x", 1.0, 2.0)])); + let t2 = Table::new("t", Some(vec![fa_f64!("x", 3.0, 4.0)])); + let st = SuperTable::from_batches(vec![Arc::new(t1), Arc::new(t2)], None); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| st.to_python(py)).unwrap(); + let Value::SuperTable(s) = result else { panic!("expected SuperTable, got {result:?}") }; + assert_eq!(s.n_rows(), 4); + println!(" SuperTable roundtrip: ok"); +} + +fn roundtrip_super_array(rt: &PyLiquid) { + let a1: Array = arr_f64![1.0, 2.0].into(); + let a2: Array = arr_f64![3.0, 4.0].into(); + let sa = SuperArray::from_arrays(vec![a1, a2]); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| sa.to_python(py)).unwrap(); + let Value::SuperArray(s) = result else { panic!("expected SuperArray, got {result:?}") }; + assert_eq!(s.len(), 4); + println!(" SuperArray roundtrip: ok"); +} + +fn roundtrip_scalar_float(rt: &PyLiquid) { + let s = Scalar::Float64(3.14); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| s.to_python(py)).unwrap(); + let Value::Scalar(Scalar::Float64(v)) = result else { panic!("expected Float64, got {result:?}") }; + assert!((v - 3.14).abs() < 1e-10); + println!(" Scalar::Float64 roundtrip: ok"); +} + +fn roundtrip_scalar_int(rt: &PyLiquid) { + let s = Scalar::Int64(42); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| s.to_python(py)).unwrap(); + let Value::Scalar(Scalar::Int64(v)) = result else { panic!("expected Int64, got {result:?}") }; + assert_eq!(v, 42); + println!(" Scalar::Int64 roundtrip: ok"); +} + +fn roundtrip_scalar_bool(rt: &PyLiquid) { + let s = Scalar::Boolean(true); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| s.to_python(py)).unwrap(); + let Value::Scalar(Scalar::Boolean(v)) = result else { panic!("expected Boolean, got {result:?}") }; + assert!(v); + println!(" Scalar::Boolean roundtrip: ok"); +} + +fn roundtrip_scalar_string(rt: &PyLiquid) { + let s = Scalar::String32("hello".to_string()); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| s.to_python(py)).unwrap(); + let Value::Scalar(Scalar::String32(v)) = result else { panic!("expected String32, got {result:?}") }; + assert_eq!(v, "hello"); + println!(" Scalar::String32 roundtrip: ok"); +} + +fn roundtrip_scalar_null(rt: &PyLiquid) { + let s = Scalar::Null; + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| s.to_python(py)).unwrap(); + let Value::Scalar(Scalar::Null) = result else { panic!("expected Null, got {result:?}") }; + println!(" Scalar::Null roundtrip: ok"); +} + +fn roundtrip_value_table(rt: &PyLiquid) { + let table = Table::new("v", Some(vec![fa_f64!("a", 1.0)])); + let value = Value::Table(Arc::new(table)); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| value.to_python(py)).unwrap(); + let Value::Table(t) = result else { panic!("expected Table, got {result:?}") }; + assert_eq!(t.n_rows, 1); + println!(" Value::Table roundtrip: ok"); +} + +fn roundtrip_value_array(rt: &PyLiquid) { + let array: Array = arr_f64![9.0, 8.0].into(); + let value = Value::Array(Arc::new(array)); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| value.to_python(py)).unwrap(); + let Value::Array(a) = result else { panic!("expected Array, got {result:?}") }; + assert_eq!(a.len(), 2); + println!(" Value::Array roundtrip: ok"); +} + +fn roundtrip_value_scalar(rt: &PyLiquid) { + let value = Value::Scalar(Scalar::Int64(99)); + let empty = Table::new_empty(); + let result = rt.with_python(&empty, |py, _| value.to_python(py)).unwrap(); + let Value::Scalar(Scalar::Int64(v)) = result else { panic!("expected Int64, got {result:?}") }; + assert_eq!(v, 99); + println!(" Value::Scalar roundtrip: ok"); +} diff --git a/minarrow-py/src/pyliquid.rs b/minarrow-py/src/pyliquid.rs index 204656f..2b6296e 100644 --- a/minarrow-py/src/pyliquid.rs +++ b/minarrow-py/src/pyliquid.rs @@ -75,7 +75,7 @@ use std::sync::Once; use minarrow::ffi::arrow_c_ffi::{ArrowArrayStream, ArrowSchema}; #[cfg(feature = "ndarray")] use minarrow::{NdArray, Vec64}; -use minarrow::{Array, Scalar, Table, Value}; +use minarrow::{Array, ArrayV, FieldArray, Scalar, SuperArray, SuperTable, Table, TableV, Value}; #[cfg(feature = "ndarray")] use minarrow_pyo3::ffi::dlpack::import_dlpack; use minarrow_pyo3::ffi::to_rust; @@ -185,6 +185,166 @@ impl PyInput for NdArray { } } +impl PyInput for FieldArray { + fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + self.array.clone().to_python(py) + } +} + +impl PyInput for TableV { + fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + let obj = PyTable(PyTableInner::from(self.clone())); + Ok(Bound::new(py, obj)?.into_any()) + } +} + +impl PyInput for ArrayV { + fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + let obj = PyArray(PyArrayInner::from(self.clone())); + Ok(Bound::new(py, obj)?.into_any()) + } +} + +impl PyInput for SuperTable { + fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + let obj = PyChunkedTable(Arc::new(self.clone()), None); + Ok(Bound::new(py, obj)?.into_any()) + } +} + +impl PyInput for SuperArray { + fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + let obj = PyChunkedArray(Arc::new(self.clone())); + Ok(Bound::new(py, obj)?.into_any()) + } +} + +impl PyInput for Scalar { + fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + use pyo3::IntoPyObject; + match self { + Scalar::Null => Ok(py.None().into_bound(py)), + Scalar::Boolean(v) => Ok(PyBool::new(py, *v).to_owned().into_any()), + #[cfg(feature = "extended_numeric_types")] + Scalar::Int8(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "extended_numeric_types")] + Scalar::Int16(v) => Ok(v.into_pyobject(py)?.into_any()), + Scalar::Int32(v) => Ok(v.into_pyobject(py)?.into_any()), + Scalar::Int64(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "extended_numeric_types")] + Scalar::UInt8(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "extended_numeric_types")] + Scalar::UInt16(v) => Ok(v.into_pyobject(py)?.into_any()), + Scalar::UInt32(v) => Ok(v.into_pyobject(py)?.into_any()), + Scalar::UInt64(v) => Ok(v.into_pyobject(py)?.into_any()), + Scalar::Float32(v) => Ok(v.into_pyobject(py)?.into_any()), + Scalar::Float64(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "decimal")] + Scalar::Decimal32(v, _precision, scale) => { + decimal_to_python(py, *v as i128, *scale) + } + #[cfg(feature = "decimal")] + Scalar::Decimal64(v, _precision, scale) => { + decimal_to_python(py, *v as i128, *scale) + } + #[cfg(feature = "decimal")] + Scalar::Decimal128(v, _precision, scale) => { + decimal_to_python(py, *v, *scale) + } + Scalar::String32(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "large_string")] + Scalar::String64(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "datetime")] + Scalar::Datetime32(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "datetime")] + Scalar::Datetime64(v) => Ok(v.into_pyobject(py)?.into_any()), + #[cfg(feature = "datetime")] + Scalar::Interval => Ok(py.None().into_bound(py)), + } + } +} + +impl PyInput for Value { + fn to_python<'py>(&self, py: Python<'py>) -> PyResult> { + match self { + Value::Scalar(s) => s.to_python(py), + Value::Array(a) => a.as_ref().to_python(py), + Value::ArrayView(av) => av.as_ref().to_python(py), + Value::FieldArray(fa) => fa.as_ref().to_python(py), + Value::Table(t) => t.as_ref().to_python(py), + Value::TableView(tv) => tv.as_ref().to_python(py), + Value::SuperArray(sa) => sa.as_ref().to_python(py), + Value::SuperArrayView(_) => Ok(py.None().into_bound(py)), // unimplemented + Value::SuperTable(st) => st.as_ref().to_python(py), + Value::SuperTableView(_) => Ok(py.None().into_bound(py)), // unimplemented + Value::Matrix(m) => { + let obj = crate::matrix::PyMatrix((**m).clone()); + Ok(Bound::new(py, obj)?.into_any()) + } + Value::MatrixView(_) => Ok(py.None().into_bound(py)), // unimplemented + Value::NdArray(nd) => nd.as_ref().to_python(py), + Value::NdArrayView(ndv) => ndv.to_ndarray().to_python(py), + Value::SuperNdArray(_) => Ok(py.None().into_bound(py)), // unimplemented + Value::SuperNdArrayView(_) => Ok(py.None().into_bound(py)), // unimplemented + Value::XArray(xa) => { + let obj = crate::xarray::PyXArray(crate::xarray::PyXArrayInner::F64(xa.clone())); + Ok(Bound::new(py, obj)?.into_any()) + } + Value::Cube(c) => { + let obj = crate::cube::PyCube(c.clone()); + Ok(Bound::new(py, obj)?.into_any()) + } + Value::VecValue(items) => { + let list = PyList::new(py, items.iter().map(|v| v.to_python(py)).collect::>>()?)?; + Ok(list.into_any()) + } + Value::BoxValue(inner) => inner.to_python(py), + Value::ArcValue(inner) => inner.as_ref().to_python(py), + Value::Tuple2(t) => { + let elements = vec![t.0.to_python(py)?, t.1.to_python(py)?]; + Ok(PyTuple::new(py, elements)?.into_any()) + } + Value::Tuple3(t) => { + let elements = vec![t.0.to_python(py)?, t.1.to_python(py)?, t.2.to_python(py)?]; + Ok(PyTuple::new(py, elements)?.into_any()) + } + Value::Tuple4(t) => { + let elements = vec![t.0.to_python(py)?, t.1.to_python(py)?, t.2.to_python(py)?, t.3.to_python(py)?]; + Ok(PyTuple::new(py, elements)?.into_any()) + } + Value::Tuple5(t) => { + let elements = vec![t.0.to_python(py)?, t.1.to_python(py)?, t.2.to_python(py)?, t.3.to_python(py)?, t.4.to_python(py)?]; + Ok(PyTuple::new(py, elements)?.into_any()) + } + Value::Tuple6(t) => { + let elements = vec![t.0.to_python(py)?, t.1.to_python(py)?, t.2.to_python(py)?, t.3.to_python(py)?, t.4.to_python(py)?, t.5.to_python(py)?]; + Ok(PyTuple::new(py, elements)?.into_any()) + } + Value::Custom(_) => Ok(py.None().into_bound(py)), + } + } +} + +/// Converts an unscaled decimal integer to a Python `decimal.Decimal`. +#[cfg(feature = "decimal")] +fn decimal_to_python<'py>( + py: Python<'py>, + unscaled: i128, + scale: i8, +) -> PyResult> { + let decimal_mod = py.import("decimal")?; + let text = if scale <= 0 { + let multiplier = 10i128.pow((-scale) as u32); + format!("{}", unscaled * multiplier) + } else { + let divisor = 10i128.pow(scale as u32); + let whole = unscaled / divisor; + let frac = (unscaled % divisor).unsigned_abs(); + format!("{whole}.{frac:0>width$}", width = scale as usize) + }; + decimal_mod.call_method1("Decimal", (text,)) +} + /// Converts a supported Python object to a Minarrow [`Value`]. /// /// Lists and tuples are processed recursively. Arrow-compatible objects are From ed95ae75f794bde243a831c7e454843436595321 Mon Sep 17 00:00:00 2001 From: Peter Bower <37089506+pbower@users.noreply.github.com> Date: Mon, 21 Sep 2026 04:51:03 +0100 Subject: [PATCH 2/2] Update pyliquid.rs --- minarrow-py/src/pyliquid.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/minarrow-py/src/pyliquid.rs b/minarrow-py/src/pyliquid.rs index 2b6296e..888274a 100644 --- a/minarrow-py/src/pyliquid.rs +++ b/minarrow-py/src/pyliquid.rs @@ -326,6 +326,8 @@ impl PyInput for Value { } /// Converts an unscaled decimal integer to a Python `decimal.Decimal`. +/// +/// This is only for the string display it does not end up widening the underlying type #[cfg(feature = "decimal")] fn decimal_to_python<'py>( py: Python<'py>,