diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9387c3a..d3a7897 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2048,9 +2048,7 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minarrow" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53f5031b7d16aa76bc8671171814419694c687f5160917a3668f010dab2b609" +version = "0.18.2" dependencies = [ "arrow", "arrow-schema", @@ -4412,9 +4410,9 @@ dependencies = [ [[package]] name = "vec64" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef6bbef159f21ac387220ebc71141524423f7886afd390bc7b140f12630425" +checksum = "54db55dbf75205d7a52f11a60aa6742d0ce6a9eb3ed97346556f26aec246f139" dependencies = [ "libc", ] diff --git a/rust/src/models/decoders/csv.rs b/rust/src/models/decoders/csv.rs index e0af00c..bfed083 100644 --- a/rust/src/models/decoders/csv.rs +++ b/rust/src/models/decoders/csv.rs @@ -913,12 +913,8 @@ fn build_categorical_col_inline( codes[r] = code; } Ok(Array::TextArray(TextArray::Categorical32( - minarrow::CategoricalArray { - data: Buffer::from(codes), - unique_values: uniques.into(), - null_mask: Some(mask_to_bitmask(null_bools)), - } - .into(), + minarrow::CategoricalArray::new(codes, uniques.into(), Some(mask_to_bitmask(null_bools))) + .into(), ))) } @@ -961,12 +957,8 @@ fn build_categorical_col_inline( codes[r] = code; } Ok(Array::TextArray(TextArray::Categorical8( - minarrow::CategoricalArray { - data: Buffer::from(codes), - unique_values: uniques.into(), - null_mask: Some(mask_to_bitmask(null_bools)), - } - .into(), + minarrow::CategoricalArray::new(codes, uniques.into(), Some(mask_to_bitmask(null_bools))) + .into(), ))) } diff --git a/rust/src/models/decoders/ipc/parser.rs b/rust/src/models/decoders/ipc/parser.rs index b3f7d88..b84802a 100644 --- a/rust/src/models/decoders/ipc/parser.rs +++ b/rust/src/models/decoders/ipc/parser.rs @@ -1553,35 +1553,35 @@ fn make_categorical_array( feature = "extended_categorical" ))] CategoricalIndexType::UInt32 => { - Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray { - data: minarrow::Buffer::from_shared(idx_data), + Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray::new( + minarrow::Buffer::from_shared(idx_data), unique_values, null_mask, - }))) + )))) } #[cfg(feature = "default_categorical_8")] CategoricalIndexType::UInt8 => { - Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray { - data: minarrow::Buffer::from_shared(idx_data), + Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray::new( + minarrow::Buffer::from_shared(idx_data), unique_values, null_mask, - }))) + )))) } #[cfg(feature = "extended_categorical")] CategoricalIndexType::UInt16 => { - Array::TextArray(TextArray::Categorical16(Arc::new(CategoricalArray { - data: minarrow::Buffer::from_shared(idx_data), + Array::TextArray(TextArray::Categorical16(Arc::new(CategoricalArray::new( + minarrow::Buffer::from_shared(idx_data), unique_values, null_mask, - }))) + )))) } #[cfg(feature = "extended_categorical")] CategoricalIndexType::UInt64 => { - Array::TextArray(TextArray::Categorical64(Arc::new(CategoricalArray { - data: minarrow::Buffer::from_shared(idx_data), + Array::TextArray(TextArray::Categorical64(Arc::new(CategoricalArray::new( + minarrow::Buffer::from_shared(idx_data), unique_values, null_mask, - }))) + )))) } #[allow(unreachable_patterns)] _ => { diff --git a/rust/src/models/decoders/json/builder.rs b/rust/src/models/decoders/json/builder.rs index 273e95c..712c4cf 100644 --- a/rust/src/models/decoders/json/builder.rs +++ b/rust/src/models/decoders/json/builder.rs @@ -503,7 +503,7 @@ mod tests { )); match &fa.array { Array::TextArray(TextArray::Categorical8(arr)) => { - assert_eq!(arr.unique_values.len(), 3); + assert_eq!(arr.unique_values().len(), 3); assert_eq!(arr.data.as_ref(), &[0u8, 1, 0, 2, 1, 0]); } _ => panic!(), diff --git a/rust/src/models/encoders/csv.rs b/rust/src/models/encoders/csv.rs index f4635c7..4fb33cd 100644 --- a/rust/src/models/encoders/csv.rs +++ b/rust/src/models/encoders/csv.rs @@ -375,7 +375,7 @@ pub fn encode_table_csv( // dictionary lookup - always clean UTF-8 let idx = arr.data.as_ref()[row] as usize; let s = arr - .unique_values + .unique_values() .get(idx) .map(String::as_str) .unwrap_or(""); @@ -385,7 +385,7 @@ pub fn encode_table_csv( Array::TextArray(TextArray::Categorical8(arr)) => { let idx = arr.data.as_ref()[row] as usize; let s = arr - .unique_values + .unique_values() .get(idx) .map(String::as_str) .unwrap_or(""); @@ -395,7 +395,7 @@ pub fn encode_table_csv( Array::TextArray(TextArray::Categorical16(arr)) => { let idx = arr.data.as_ref()[row] as usize; let s = arr - .unique_values + .unique_values() .get(idx) .map(String::as_str) .unwrap_or(""); @@ -405,7 +405,7 @@ pub fn encode_table_csv( Array::TextArray(TextArray::Categorical64(arr)) => { let idx = arr.data.as_ref()[row] as usize; let s = arr - .unique_values + .unique_values() .get(idx) .map(String::as_str) .unwrap_or(""); diff --git a/rust/src/models/encoders/ipc/table_stream.rs b/rust/src/models/encoders/ipc/table_stream.rs index 2b21bdf..5a5513f 100644 --- a/rust/src/models/encoders/ipc/table_stream.rs +++ b/rust/src/models/encoders/ipc/table_stream.rs @@ -273,11 +273,11 @@ mod tests { None, ); - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u32, 0, 2, 1])), - unique_values: Vec64::from(dict_strs()), - null_mask: Some(make_bitmask(&[true, false, true, true])), - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u32, 0, 2, 1])), + Vec64::from(dict_strs()), + Some(make_bitmask(&[true, false, true, true])), + ); writer.register_dictionary(0, dict_strs()); @@ -322,11 +322,11 @@ mod tests { None, ); - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), - unique_values: Vec64::from(dict_strs()), - null_mask: Some(make_bitmask(&[true, false, true, true])), - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), + Vec64::from(dict_strs()), + Some(make_bitmask(&[true, false, true, true])), + ); writer.register_dictionary(0, dict_strs()); @@ -371,11 +371,11 @@ mod tests { None, ); - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), - unique_values: Vec64::from(dict_strs()), - null_mask: Some(make_bitmask(&[true, true, false, true])), - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), + Vec64::from(dict_strs()), + Some(make_bitmask(&[true, true, false, true])), + ); writer.register_dictionary(0, dict_strs()); @@ -416,11 +416,11 @@ mod tests { None, ); - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[2u16, 1, 0, 2])), - unique_values: Vec64::from(dict_strs()), - null_mask: None, - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[2u16, 1, 0, 2])), + Vec64::from(dict_strs()), + None, + ); writer.register_dictionary(0, dict_strs()); @@ -461,11 +461,11 @@ mod tests { None, ); - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[0u64, 2, 1, 0])), - unique_values: Vec64::from(dict_strs()), - null_mask: None, - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[0u64, 2, 1, 0])), + Vec64::from(dict_strs()), + None, + ); writer.register_dictionary(0, dict_strs()); @@ -506,11 +506,11 @@ mod tests { None, ); - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[0u32, 1, 1, 2])), - unique_values: Vec64::from(dict_strs()), - null_mask: Some(make_bitmask(&[true, false, true, true])), - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[0u32, 1, 1, 2])), + Vec64::from(dict_strs()), + Some(make_bitmask(&[true, false, true, true])), + ); writer.register_dictionary(0, dict_strs()); @@ -570,11 +570,11 @@ mod tests { None, ); - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[0u8, 1, 1, 2])), - unique_values: Vec64::from(dict_strs()), - null_mask: Some(make_bitmask(&[true, false, true, true])), - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[0u8, 1, 1, 2])), + Vec64::from(dict_strs()), + Some(make_bitmask(&[true, false, true, true])), + ); writer.register_dictionary(0, dict_strs()); diff --git a/rust/src/models/encoders/json.rs b/rust/src/models/encoders/json.rs index 907e4c0..fc88121 100644 --- a/rust/src/models/encoders/json.rs +++ b/rust/src/models/encoders/json.rs @@ -240,19 +240,19 @@ fn collect_cat_maps(table: &Table) -> Vec> { feature = "extended_categorical" ))] Array::TextArray(TextArray::Categorical32(arr)) => { - cat_maps.push(Some(&arr.unique_values)) + cat_maps.push(Some(&arr.unique_values())) } #[cfg(feature = "default_categorical_8")] Array::TextArray(TextArray::Categorical8(arr)) => { - cat_maps.push(Some(&arr.unique_values)) + cat_maps.push(Some(&arr.unique_values())) } #[cfg(feature = "extended_categorical")] Array::TextArray(TextArray::Categorical16(arr)) => { - cat_maps.push(Some(&arr.unique_values)) + cat_maps.push(Some(&arr.unique_values())) } #[cfg(feature = "extended_categorical")] Array::TextArray(TextArray::Categorical64(arr)) => { - cat_maps.push(Some(&arr.unique_values)) + cat_maps.push(Some(&arr.unique_values())) } _ => cat_maps.push(None), } diff --git a/rust/src/models/readers/chunked/parquet.rs b/rust/src/models/readers/chunked/parquet.rs index 61a4505..270fdfd 100644 --- a/rust/src/models/readers/chunked/parquet.rs +++ b/rust/src/models/readers/chunked/parquet.rs @@ -189,11 +189,11 @@ mod tests { let indices: Vec64 = (0..n_rows).map(|i| (i % 3) as u8).collect(); ( ArrowType::Dictionary(CategoricalIndexType::UInt8), - Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray { - data: Buffer::from(indices), + Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray::new( + Buffer::from(indices), unique_values, - null_mask: Some(Bitmask::new_set_all(n_rows, true)), - }))), + Some(Bitmask::new_set_all(n_rows, true)), + )))), ) }; #[cfg(not(feature = "default_categorical_8"))] @@ -201,11 +201,11 @@ mod tests { let indices: Vec64 = (0..n_rows).map(|i| (i % 3) as u32).collect(); ( ArrowType::Dictionary(CategoricalIndexType::UInt32), - Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray { - data: Buffer::from(indices), + Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray::new( + Buffer::from(indices), unique_values, - null_mask: Some(Bitmask::new_set_all(n_rows, true)), - }))), + Some(Bitmask::new_set_all(n_rows, true)), + )))), ) }; let dict_col = FieldArray::new( diff --git a/rust/src/models/readers/ipc/table.rs b/rust/src/models/readers/ipc/table.rs index 8b10d68..fca95da 100644 --- a/rust/src/models/readers/ipc/table.rs +++ b/rust/src/models/readers/ipc/table.rs @@ -89,15 +89,9 @@ impl TableReader { n: Option, ) -> io::Result { let mut batches = Vec::new(); - let mut schema: Option>> = None; - let mut n_rows = 0usize; let mut count = 0usize; while let Some(batch) = self.next().await { let batch = batch?; - if schema.is_none() { - schema = Some(batch.cols.iter().map(|f| f.field.clone()).collect()); - } - n_rows += batch.n_rows; batches.push(Arc::new(batch)); count += 1; if let Some(max) = n @@ -105,12 +99,10 @@ impl TableReader { break; } } - Ok(SuperTable { + Ok(SuperTable::from_batches( batches, - schema: schema.unwrap_or_default(), - n_rows, - name: name.unwrap_or_else(|| "SuperTable".to_string()), - }) + Some(name.unwrap_or_else(|| "SuperTable".to_string())), + )) } /// Read all batches and concatenate into a single `Table` row-wise. diff --git a/rust/src/models/readers/ipc/window.rs b/rust/src/models/readers/ipc/window.rs index 1c85e82..0e3557a 100644 --- a/rust/src/models/readers/ipc/window.rs +++ b/rust/src/models/readers/ipc/window.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use minarrow::{ Array, Bitmask, BooleanArray, Buffer, CategoricalArray, FieldArray, FloatArray, Integer, - IntegerArray, NumericArray, StringArray, Table, TextArray, Vec64, + IntegerArray, MaskedArray, NumericArray, StringArray, Table, TextArray, Vec64, }; #[cfg(feature = "decimal")] use minarrow::DecimalArray; @@ -225,11 +225,7 @@ fn window_categorical( offset: usize, len: usize, ) -> CategoricalArray { - CategoricalArray { - data: window_buffer(&arr.data, offset, len), - unique_values: arr.unique_values.clone(), - null_mask: window_mask(arr.null_mask.as_ref(), offset, len), - } + arr.slice_clone(offset, len) } /// Window a buffer to elements `[offset, offset + len)`. Shared-backed diff --git a/rust/src/models/readers/parquet.rs b/rust/src/models/readers/parquet.rs index c461e20..0268266 100644 --- a/rust/src/models/readers/parquet.rs +++ b/rust/src/models/readers/parquet.rs @@ -58,7 +58,7 @@ use crate::models::types::parquet::{ ParquetEncoding, ParquetLogicalType, ParquetPhysicalType, parquet_to_arrow_type, }; use minarrow::{ - Array, ArrowType, Bitmask, BooleanArray, Field, FieldArray, FloatArray, IntegerArray, + Array, ArrowType, Bitmask, BooleanArray, CategoricalArray, Field, FieldArray, FloatArray, IntegerArray, NumericArray, StringArray, Table, TextArray, Vec64, vec64, }; #[cfg(feature = "decimal")] @@ -749,6 +749,53 @@ fn decode_column( }) } +// categorical builders + +#[cfg(any( + not(feature = "default_categorical_8"), + feature = "extended_categorical" +))] +fn build_cat32(idx: Vec64, dict_raw: &[Vec], mask: Option) -> Array { + let dict = dict_raw + .iter() + .map(|b| String::from_utf8(b.clone()).unwrap()) + .collect::>() + .into(); + Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray::new( + idx, + dict, + mask, + )))) +} + +#[cfg(feature = "default_categorical_8")] +fn build_cat8(idx: Vec64, dict_raw: &[Vec], mask: Option) -> Array { + let dict = dict_raw + .iter() + .map(|b| String::from_utf8(b.clone()).unwrap()) + .collect::>(); + let idx8: Vec64 = idx.iter().map(|&v| v as u8).collect(); + Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray::new( + idx8, + dict, + mask, + )))) +} + +#[cfg(all(feature = "extended_categorical", feature = "large_string"))] +fn build_cat64(idx: Vec64, dict_raw: &[Vec], mask: Option) -> Array { + let dict = dict_raw + .iter() + .map(|b| String::from_utf8(b.clone()).unwrap()) + .collect::>() + .into(); + Array::TextArray(TextArray::Categorical64(Arc::new(CategoricalArray::new( + idx, + dict, + mask, + )))) +} + // RLE/bit-packed Hybrid decoder fn decode_hybrid(buf: &[u8], bit_width: u8, n: usize) -> Result, IoError> { @@ -1630,6 +1677,32 @@ mod tests { let out = super::decode_hybrid(buf, bit_width, expect.len()).unwrap(); assert_eq!(out.as_slice(), expect.as_slice()); } + + // Dictionary encoding is a page storage detail: the reader expands it + // before decode_column and preserves the schema's UTF8 column type. + #[cfg(feature = "snappy")] + #[test] + fn read_dictionary_encoded_strings() { + use minarrow::MaskedArray; + + let bytes = include_bytes!("../../../pyarrow-roundtrip/pyarrow_simple.parquet"); + let table = load_parquet_table(Cursor::new(bytes.as_slice())).unwrap(); + assert_eq!(table.n_rows, 5); + let column = &table.cols[1]; + assert_eq!(column.field.name, "name"); + assert_eq!(column.field.dtype, ArrowType::String); + match &column.array { + Array::TextArray(TextArray::String32(values)) => { + let decoded: Vec<_> = (0..table.n_rows).map(|i| values.get(i)).collect(); + assert_eq!( + decoded, + [Some("Alice"), Some("Bob"), Some("Charlie"), Some("Diana"), Some("Eve")] + ); + assert_eq!(column.null_count, 0); + } + other => panic!("expected decoded UTF8 strings, got {other:?}"), + } + } #[test] fn decode_column_plain_int32() { diff --git a/rust/src/models/writers/ipc/table.rs b/rust/src/models/writers/ipc/table.rs index 7ef0056..6fdcfbf 100644 --- a/rust/src/models/writers/ipc/table.rs +++ b/rust/src/models/writers/ipc/table.rs @@ -218,11 +218,11 @@ mod tests { #[cfg(not(feature = "default_categorical_8"))] fn make_table() -> Table { - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u32, 0, 2, 1])), - unique_values: Vec64::from(dict_strs()), - null_mask: Some(make_bitmask(&[true, false, true, true])), - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u32, 0, 2, 1])), + Vec64::from(dict_strs()), + Some(make_bitmask(&[true, false, true, true])), + ); Table { cols: vec![FieldArray::new( Field { @@ -242,11 +242,11 @@ mod tests { #[cfg(feature = "default_categorical_8")] fn make_table() -> Table { - let arr = CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), - unique_values: Vec64::from(dict_strs()), - null_mask: Some(make_bitmask(&[true, false, true, true])), - }; + let arr = CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), + Vec64::from(dict_strs()), + Some(make_bitmask(&[true, false, true, true])), + ); Table { cols: vec![FieldArray::new( Field { diff --git a/rust/src/models/writers/parquet.rs b/rust/src/models/writers/parquet.rs index bbf5cd6..3ca7489 100644 --- a/rust/src/models/writers/parquet.rs +++ b/rust/src/models/writers/parquet.rs @@ -186,7 +186,7 @@ pub fn write_parquet_table( let (dict_uncomp, dict_comp) = write_dictionary_page( &mut out, &mut offset, - a.unique_values.iter().map(|s| s.as_bytes()), + a.unique_values().iter().map(|s| s.as_bytes()), compression, )?; total_uncompressed_size += dict_uncomp; @@ -197,7 +197,7 @@ pub fn write_parquet_table( let (dict_uncomp, dict_comp) = write_dictionary_page( &mut out, &mut offset, - a.unique_values.iter().map(|s| s.as_bytes()), + a.unique_values().iter().map(|s| s.as_bytes()), compression, )?; total_uncompressed_size += dict_uncomp; diff --git a/rust/src/test_helpers.rs b/rust/src/test_helpers.rs index 9fb85c1..ea18e9e 100644 --- a/rust/src/test_helpers.rs +++ b/rust/src/test_helpers.rs @@ -179,15 +179,15 @@ pub(crate) fn dict32_col() -> FieldArray { nullable: true, metadata: Default::default(), }, - Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u32, 0, 2, 1])), - unique_values: Vec64::from(vec![ + Array::TextArray(TextArray::Categorical32(Arc::new(CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u32, 0, 2, 1])), + Vec64::from(vec![ "apple".to_string(), "banana".to_string(), "pear".to_string(), ]), - null_mask: Some(Bitmask::new_set_all(4, true)), - }))), + Some(Bitmask::new_set_all(4, true)), + )))), ) } @@ -201,15 +201,15 @@ pub(crate) fn dict8_default_col() -> FieldArray { nullable: true, metadata: Default::default(), }, - Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), - unique_values: Vec64::from(vec![ + Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), + Vec64::from(vec![ "apple".to_string(), "banana".to_string(), "pear".to_string(), ]), - null_mask: Some(Bitmask::new_set_all(4, true)), - }))), + Some(Bitmask::new_set_all(4, true)), + )))), ) } @@ -280,11 +280,11 @@ pub(crate) fn dict8_extended_col() -> FieldArray { nullable: true, metadata: Default::default(), }, - Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), - unique_values: Vec64::from(vec!["a".to_string(), "b".to_string(), "c".to_string()]), - null_mask: Some(Bitmask::new_set_all(4, true)), - }))), + Array::TextArray(TextArray::Categorical8(Arc::new(CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u8, 0, 2, 1])), + Vec64::from(vec!["a".to_string(), "b".to_string(), "c".to_string()]), + Some(Bitmask::new_set_all(4, true)), + )))), ) } @@ -299,11 +299,11 @@ pub(crate) fn dict16_col() -> FieldArray { nullable: true, metadata: Default::default(), }, - Array::TextArray(TextArray::Categorical16(Arc::new(CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u16, 0, 2, 1])), - unique_values: Vec64::from(vec!["x".to_string(), "y".to_string(), "z".to_string()]), - null_mask: Some(Bitmask::new_set_all(4, true)), - }))), + Array::TextArray(TextArray::Categorical16(Arc::new(CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u16, 0, 2, 1])), + Vec64::from(vec!["x".to_string(), "y".to_string(), "z".to_string()]), + Some(Bitmask::new_set_all(4, true)), + )))), ) } @@ -318,11 +318,11 @@ pub(crate) fn dict64_col() -> FieldArray { nullable: true, metadata: Default::default(), }, - Array::TextArray(TextArray::Categorical64(Arc::new(CategoricalArray { - data: Buffer::from(Vec64::from_slice(&[1u64, 0, 2, 1])), - unique_values: Vec64::from(vec!["long".to_string(), "lo".to_string(), "l".to_string()]), - null_mask: Some(Bitmask::new_set_all(4, true)), - }))), + Array::TextArray(TextArray::Categorical64(Arc::new(CategoricalArray::new( + Buffer::from(Vec64::from_slice(&[1u64, 0, 2, 1])), + Vec64::from(vec!["long".to_string(), "lo".to_string(), "l".to_string()]), + Some(Bitmask::new_set_all(4, true)), + )))), ) } diff --git a/rust/src/utils.rs b/rust/src/utils.rs index 3c2f2e6..edf03a0 100644 --- a/rust/src/utils.rs +++ b/rust/src/utils.rs @@ -99,19 +99,19 @@ pub(crate) fn dict_values(array: &minarrow::Array) -> Option> { feature = "extended_categorical" ))] minarrow::Array::TextArray(Categorical32(arr)) => { - Some(arr.unique_values.iter().cloned().collect()) + Some(arr.unique_values().iter().cloned().collect()) } #[cfg(feature = "default_categorical_8")] minarrow::Array::TextArray(Categorical8(arr)) => { - Some(arr.unique_values.iter().cloned().collect()) + Some(arr.unique_values().iter().cloned().collect()) } #[cfg(feature = "extended_categorical")] minarrow::Array::TextArray(Categorical16(arr)) => { - Some(arr.unique_values.iter().cloned().collect()) + Some(arr.unique_values().iter().cloned().collect()) } #[cfg(feature = "extended_categorical")] minarrow::Array::TextArray(Categorical64(arr)) => { - Some(arr.unique_values.iter().cloned().collect()) + Some(arr.unique_values().iter().cloned().collect()) } _ => None, }