From 9c39365ca28d04f36f8075c25537c3485896b1b2 Mon Sep 17 00:00:00 2001 From: JunRuiLee Date: Mon, 27 Jul 2026 13:55:05 +0800 Subject: [PATCH 1/2] feat(c): construct table from resolved schema --- Cargo.lock | 1 + bindings/c/Cargo.toml | 1 + bindings/c/src/table.rs | 172 ++++++++++++++++++++++++++++++- bindings/c/src/tests.rs | 181 +++++++++++++++++++++++++++++++++ crates/paimon/src/table/mod.rs | 35 +++++++ 5 files changed, 385 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ee0257e..47eb4598 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4595,6 +4595,7 @@ dependencies = [ "futures", "paimon", "paimon-vindex-core", + "serde_json", "tokio", ] diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml index f02c3e83..2340f970 100644 --- a/bindings/c/Cargo.toml +++ b/bindings/c/Cargo.toml @@ -37,6 +37,7 @@ futures = "0.3" arrow = { workspace = true } arrow-array = { workspace = true } arrow-schema = { workspace = true } +serde_json = "1.0.120" [dev-dependencies] # Test-only: the vector-search integration tests build a real primary-key vindex diff --git a/bindings/c/src/table.rs b/bindings/c/src/table.rs index 05f82e31..3ed89546 100644 --- a/bindings/c/src/table.rs +++ b/bindings/c/src/table.rs @@ -16,19 +16,22 @@ // under the License. use std::collections::HashMap; -use std::ffi::c_void; +use std::ffi::{c_char, c_void}; use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use arrow_array::{Array, StructArray}; use futures::StreamExt; -use paimon::spec::{DataField, DataType, Datum, Predicate, PredicateBuilder}; +use paimon::catalog::Identifier; +use paimon::io::FileIO; +use paimon::spec::{DataField, DataType, Datum, Predicate, PredicateBuilder, TableSchema}; use paimon::table::{ArrowRecordBatchStream, DataSplit, Table}; use paimon::Plan; use crate::error::{check_non_null, paimon_error, validate_cstr, PaimonErrorCode}; use crate::result::{ - paimon_result_new_read, paimon_result_next_batch, paimon_result_plan, paimon_result_predicate, - paimon_result_read_builder, paimon_result_record_batch_reader, paimon_result_table_scan, + paimon_result_get_table, paimon_result_new_read, paimon_result_next_batch, paimon_result_plan, + paimon_result_predicate, paimon_result_read_builder, paimon_result_record_batch_reader, + paimon_result_table_scan, }; use crate::runtime; use crate::types::*; @@ -58,10 +61,160 @@ unsafe fn box_table_read_state(state: TableReadState) -> *mut paimon_table_read // ======================= Table =============================== +/// Create a table directly from a resolved Paimon table schema JSON. +/// +/// This constructor does not create a catalog or derive a warehouse. Storage +/// options are used only to build FileIO; they are not merged into the supplied +/// table schema. `branch` selects the branch-scoped managers while preserving +/// the supplied schema. +/// +/// # Safety +/// All string pointers must be valid null-terminated C strings. `storage_options` +/// must point to `storage_options_len` valid `paimon_option` values, or be null +/// when `storage_options_len` is 0. +#[no_mangle] +pub unsafe extern "C" fn paimon_table_from_schema_json( + table_path: *const c_char, + table_schema_json: *const c_char, + database: *const c_char, + table_name: *const c_char, + branch: *const c_char, + storage_options: *const paimon_option, + storage_options_len: usize, +) -> paimon_result_get_table { + let table_path = match validate_cstr(table_path, "table_path") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } + } + }; + let table_schema_json = match validate_cstr(table_schema_json, "table_schema_json") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } + } + }; + let database = match validate_cstr(database, "database") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } + } + }; + let table_name = match validate_cstr(table_name, "table_name") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } + } + }; + let branch = match validate_cstr(branch, "branch") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } + } + }; + if storage_options.is_null() && storage_options_len > 0 { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + "null storage_options pointer with non-zero length".to_string(), + ), + }; + } + + let schema = match serde_json::from_str::(&table_schema_json) { + Ok(schema) => schema, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("Failed to parse table schema JSON: {error}"), + ), + } + } + }; + + let mut options = HashMap::with_capacity(storage_options_len); + if storage_options_len > 0 { + for option in std::slice::from_raw_parts(storage_options, storage_options_len) { + let key = match validate_cstr(option.key, "storage option key") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } + } + }; + let value = match validate_cstr(option.value, "storage option value") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } + } + }; + options.insert(key, value); + } + } + + let file_io = match FileIO::from_path(&table_path) + .and_then(|builder| builder.with_props(options.iter()).build()) + { + Ok(file_io) => file_io, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error: paimon_error::from_paimon(error), + } + } + }; + let table = match Table::from_resolved_schema( + file_io, + Identifier::new(database, table_name), + table_path, + schema, + branch, + ) { + Ok(table) => table, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error: paimon_error::from_paimon(error), + } + } + }; + let wrapper = Box::new(paimon_table { + inner: Box::into_raw(Box::new(table)) as *mut c_void, + }); + paimon_result_get_table { + table: Box::into_raw(wrapper), + error: std::ptr::null_mut(), + } +} + /// Free a paimon_table. /// /// # Safety -/// Only call with a table returned from `paimon_catalog_get_table`. +/// Only call with a table returned from `paimon_catalog_get_table` or +/// `paimon_table_from_schema_json`. #[no_mangle] pub unsafe extern "C" fn paimon_table_free(table: *mut paimon_table) { free_table_wrapper(table, |t| t.inner); @@ -1773,6 +1926,15 @@ const _: unsafe extern "C" fn( // constructors so an accidental signature change fails to compile rather than // silently breaking header consumers. To add behavior, introduce a new // `paimon_table_new_read_builder_*` symbol instead of changing one of these. +const _: unsafe extern "C" fn( + *const c_char, + *const c_char, + *const c_char, + *const c_char, + *const c_char, + *const paimon_option, + usize, +) -> paimon_result_get_table = paimon_table_from_schema_json; const _: unsafe extern "C" fn(*const paimon_table) -> paimon_result_read_builder = paimon_table_new_read_builder; const _: unsafe extern "C" fn( diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index 9c35fb98..6920b18e 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -85,6 +85,10 @@ unsafe fn unwrap_table(table: *mut paimon_table) { } } +unsafe fn table_ref<'a>(table: *const paimon_table) -> &'a Table { + &*((*table).inner as *const Table) +} + fn make_batch(ids: Vec, names: Vec<&str>) -> RecordBatch { let schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("id", ArrowDataType::Int32, false), @@ -293,6 +297,183 @@ unsafe fn read_rows_ffi(table: *const paimon_table) -> Vec<(i32, String)> { rows } +// ========================================================================= +// Catalog-free table construction tests +// ========================================================================= + +#[test] +fn test_table_from_schema_json_preserves_resolved_schema_and_branch() { + let path = CString::new("memory:/test_resolved_branch").unwrap(); + let database = CString::new("default").unwrap(); + let table_name = CString::new("test").unwrap(); + let branch = CString::new("dev").unwrap(); + let schema = simple_table_schema(); + let schema_json = CString::new(serde_json::to_string(&schema).unwrap()).unwrap(); + let option_key = CString::new("storage-only-option").unwrap(); + let option_value = CString::new("secret").unwrap(); + let storage_options = [paimon_option { + key: option_key.as_ptr(), + value: option_value.as_ptr(), + }]; + + unsafe { + let result = paimon_table_from_schema_json( + path.as_ptr(), + schema_json.as_ptr(), + database.as_ptr(), + table_name.as_ptr(), + branch.as_ptr(), + storage_options.as_ptr(), + storage_options.len(), + ); + + assert!(result.error.is_null()); + assert!(!result.table.is_null()); + let table = table_ref(result.table); + assert_eq!(table.location(), "memory:/test_resolved_branch"); + assert_eq!(table.identifier(), &Identifier::new("default", "test")); + assert_eq!(table.schema(), &schema); + assert_eq!(table.branch(), "dev"); + assert!(table.is_branch_reference()); + assert_eq!( + table.schema_manager().schema_path(schema.id()), + "memory:/test_resolved_branch/branch/branch-dev/schema/schema-0" + ); + assert_eq!( + table.snapshot_manager().snapshot_path(1), + "memory:/test_resolved_branch/branch/branch-dev/snapshot/snapshot-1" + ); + assert_eq!( + table.tag_manager().tag_path("release"), + "memory:/test_resolved_branch/branch/branch-dev/tag/tag-release" + ); + assert!(!table.schema().options().contains_key("storage-only-option")); + + let read_builder = paimon_table_new_read_builder(result.table); + assert!(read_builder.error.is_null()); + assert!(!read_builder.read_builder.is_null()); + paimon_read_builder_free(read_builder.read_builder); + + paimon_table_free(result.table); + } +} + +#[test] +fn test_table_from_schema_json_main_branch_uses_main_schema_directory() { + let path = CString::new("memory:/test_resolved_main").unwrap(); + let database = CString::new("default").unwrap(); + let table_name = CString::new("test").unwrap(); + let branch = CString::new("main").unwrap(); + let schema = simple_table_schema(); + let schema_json = CString::new(serde_json::to_string(&schema).unwrap()).unwrap(); + + unsafe { + let result = paimon_table_from_schema_json( + path.as_ptr(), + schema_json.as_ptr(), + database.as_ptr(), + table_name.as_ptr(), + branch.as_ptr(), + std::ptr::null(), + 0, + ); + + assert!(result.error.is_null()); + assert!(!result.table.is_null()); + let table = table_ref(result.table); + assert_eq!(table.branch(), "main"); + assert!(!table.is_branch_reference()); + assert_eq!( + table.schema_manager().schema_path(schema.id()), + "memory:/test_resolved_main/schema/schema-0" + ); + + paimon_table_free(result.table); + } +} + +#[test] +fn test_table_from_schema_json_rejects_invalid_input() { + let path = CString::new("memory:/test_resolved_invalid").unwrap(); + let malformed_schema = CString::new("not-json").unwrap(); + let database = CString::new("default").unwrap(); + let table_name = CString::new("test").unwrap(); + let branch = CString::new("main").unwrap(); + + unsafe { + let malformed = paimon_table_from_schema_json( + path.as_ptr(), + malformed_schema.as_ptr(), + database.as_ptr(), + table_name.as_ptr(), + branch.as_ptr(), + std::ptr::null(), + 0, + ); + assert!(malformed.table.is_null()); + assert!(!malformed.error.is_null()); + assert_eq!( + (*malformed.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(malformed.error); + + let schema = simple_table_schema(); + let schema_json = CString::new(serde_json::to_string(&schema).unwrap()).unwrap(); + let null_path = paimon_table_from_schema_json( + std::ptr::null(), + schema_json.as_ptr(), + database.as_ptr(), + table_name.as_ptr(), + branch.as_ptr(), + std::ptr::null(), + 0, + ); + assert!(null_path.table.is_null()); + assert!(!null_path.error.is_null()); + assert_eq!( + (*null_path.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(null_path.error); + + let null_options = paimon_table_from_schema_json( + path.as_ptr(), + schema_json.as_ptr(), + database.as_ptr(), + table_name.as_ptr(), + branch.as_ptr(), + std::ptr::null(), + 1, + ); + assert!(null_options.table.is_null()); + assert!(!null_options.error.is_null()); + assert_eq!( + (*null_options.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(null_options.error); + + let invalid_branch = CString::new("../dev").unwrap(); + let unsafe_branch = paimon_table_from_schema_json( + path.as_ptr(), + schema_json.as_ptr(), + database.as_ptr(), + table_name.as_ptr(), + invalid_branch.as_ptr(), + std::ptr::null(), + 0, + ); + assert!(unsafe_branch.table.is_null()); + assert!(!unsafe_branch.error.is_null()); + assert_eq!( + (*unsafe_branch.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(unsafe_branch.error); + } +} + // ========================================================================= // Read path tests // ========================================================================= diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index be680cb0..b82b7c9d 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -189,6 +189,41 @@ impl Table { } } + /// Create a table from an already resolved schema without loading a catalog. + /// + /// The supplied schema is preserved as-is. The branch only selects the + /// branch-scoped managers used by subsequent reads. + pub fn from_resolved_schema( + file_io: FileIO, + identifier: Identifier, + location: String, + schema: TableSchema, + branch: impl Into, + ) -> Result { + let branch = branch.into(); + validate_branch_name(&branch)?; + let schema_manager = SchemaManager::new(file_io.clone(), location.clone()); + let schema_manager = if branch == DEFAULT_MAIN_BRANCH { + schema_manager + } else { + schema_manager.with_branch(&branch) + }; + let branch_reference = branch != DEFAULT_MAIN_BRANCH; + + Ok(Self { + file_io, + identifier, + location, + schema, + schema_manager, + branch, + branch_reference, + rest_env: None, + time_traveled: false, + travel_snapshot: None, + }) + } + /// Get the table's identifier. pub fn identifier(&self) -> &Identifier { &self.identifier From 7636fda88c8fc41f3dd1921adf8f72cd06be1075 Mon Sep 17 00:00:00 2001 From: JunRuiLee Date: Tue, 28 Jul 2026 15:55:40 +0800 Subject: [PATCH 2/2] feat(c): default branch to main when null in table_from_schema_json Allow the branch pointer of paimon_table_from_schema_json to be null, defaulting to DEFAULT_MAIN_BRANCH ("main"). A non-null pointer still goes through the full UTF-8 and branch-name validation, so invalid names such as "../dev" are still rejected. Update the doc and Safety comment accordingly and add a test covering the null-branch default. --- bindings/c/src/table.rs | 28 +++++++++++++++++----------- bindings/c/src/tests.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/bindings/c/src/table.rs b/bindings/c/src/table.rs index 3ed89546..79d809bc 100644 --- a/bindings/c/src/table.rs +++ b/bindings/c/src/table.rs @@ -21,7 +21,7 @@ use std::ffi::{c_char, c_void}; use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use arrow_array::{Array, StructArray}; use futures::StreamExt; -use paimon::catalog::Identifier; +use paimon::catalog::{Identifier, DEFAULT_MAIN_BRANCH}; use paimon::io::FileIO; use paimon::spec::{DataField, DataType, Datum, Predicate, PredicateBuilder, TableSchema}; use paimon::table::{ArrowRecordBatchStream, DataSplit, Table}; @@ -66,12 +66,14 @@ unsafe fn box_table_read_state(state: TableReadState) -> *mut paimon_table_read /// This constructor does not create a catalog or derive a warehouse. Storage /// options are used only to build FileIO; they are not merged into the supplied /// table schema. `branch` selects the branch-scoped managers while preserving -/// the supplied schema. +/// the supplied schema; pass null to default to the `main` branch. /// /// # Safety -/// All string pointers must be valid null-terminated C strings. `storage_options` -/// must point to `storage_options_len` valid `paimon_option` values, or be null -/// when `storage_options_len` is 0. +/// All string pointers except `branch` must be valid null-terminated C strings. +/// `branch` may be null to select the default `main` branch, or a valid +/// null-terminated C string. `storage_options` must point to +/// `storage_options_len` valid `paimon_option` values, or be null when +/// `storage_options_len` is 0. #[no_mangle] pub unsafe extern "C" fn paimon_table_from_schema_json( table_path: *const c_char, @@ -118,12 +120,16 @@ pub unsafe extern "C" fn paimon_table_from_schema_json( } } }; - let branch = match validate_cstr(branch, "branch") { - Ok(value) => value, - Err(error) => { - return paimon_result_get_table { - table: std::ptr::null_mut(), - error, + let branch = if branch.is_null() { + DEFAULT_MAIN_BRANCH.to_string() + } else { + match validate_cstr(branch, "branch") { + Ok(value) => value, + Err(error) => { + return paimon_result_get_table { + table: std::ptr::null_mut(), + error, + } } } }; diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index 6920b18e..19810b39 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -392,6 +392,39 @@ fn test_table_from_schema_json_main_branch_uses_main_schema_directory() { } } +#[test] +fn test_table_from_schema_json_null_branch_defaults_to_main() { + let path = CString::new("memory:/test_resolved_null_branch").unwrap(); + let database = CString::new("default").unwrap(); + let table_name = CString::new("test").unwrap(); + let schema = simple_table_schema(); + let schema_json = CString::new(serde_json::to_string(&schema).unwrap()).unwrap(); + + unsafe { + let result = paimon_table_from_schema_json( + path.as_ptr(), + schema_json.as_ptr(), + database.as_ptr(), + table_name.as_ptr(), + std::ptr::null(), + std::ptr::null(), + 0, + ); + + assert!(result.error.is_null()); + assert!(!result.table.is_null()); + let table = table_ref(result.table); + assert_eq!(table.branch(), "main"); + assert!(!table.is_branch_reference()); + assert_eq!( + table.schema_manager().schema_path(schema.id()), + "memory:/test_resolved_null_branch/schema/schema-0" + ); + + paimon_table_free(result.table); + } +} + #[test] fn test_table_from_schema_json_rejects_invalid_input() { let path = CString::new("memory:/test_resolved_invalid").unwrap();