Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
3388c05
fix(xlsb): finish the row when WriteRow rejects a cell
GabrielMarquezMatte Nov 18, 2026
cb7ea1c
fix(netframework): typed reads skip indexer properties
GabrielMarquezMatte Nov 18, 2026
e9c899c
fix(reader): parse workbook-stored numbers invariantly whatever the c…
GabrielMarquezMatte Nov 18, 2026
350b53c
fix(parser): bind a CsvSheet held as IExcelSheet with the CSV map
GabrielMarquezMatte Nov 18, 2026
45ef3ea
fix(netframework): a new property hides the base property it shadows
GabrielMarquezMatte Nov 18, 2026
e42b44e
fix(netframework): Aggregate rethrows the failure the native run ende…
GabrielMarquezMatte Nov 18, 2026
d5cb584
fix(reader): honor the cancellation token on memory-backed XLSX and X…
GabrielMarquezMatte Nov 18, 2026
765f2c5
fix(writer): leave cell reference and pending style intact on a rejec…
GabrielMarquezMatte Nov 18, 2026
8ab8dee
fix(csv): cancel a partitioned parallel parse with the ParseAsync token
GabrielMarquezMatte Nov 18, 2026
6f6c99d
fix(netframework): number-only cell text round-trips to the same double
GabrielMarquezMatte Nov 18, 2026
ca21a5c
Merge branch 'fix/bughunt-netframework' into develop
GabrielMarquezMatte Nov 18, 2026
83fd914
fix(csv): let the parallel parse's sequential fallback be enumerated …
GabrielMarquezMatte Nov 18, 2026
3a5a9ad
fix(xls): reject a Workbook stream past 2 GiB instead of wrapping off…
GabrielMarquezMatte Nov 18, 2026
2e6d57e
fix(csv): keep the constructor token when a CsvEnumerable is enumerat…
GabrielMarquezMatte Nov 18, 2026
7819968
fix(reader): detect encrypted or oversized ZIPs as Unknown
GabrielMarquezMatte Nov 18, 2026
a16db95
fix(xls): truncate continuation sheet names on a code-point boundary
GabrielMarquezMatte Nov 18, 2026
e8da3dc
fix(csv): yield nothing from a parallel parse whose source ends befor…
GabrielMarquezMatte Nov 18, 2026
2af4fab
fix(xlsx): skip comments, PIs and DOCTYPE when detecting the element …
GabrielMarquezMatte Nov 18, 2026
1cb2b8e
fix(xlsx): read time-only t="d" values in 1904 workbooks and with a n…
GabrielMarquezMatte Nov 18, 2026
56c2390
fix(zip): match entry name prefixes ASCII case-insensitively
GabrielMarquezMatte Nov 18, 2026
7de4b12
Merge branch 'fix/bughunt-xlsx' into develop
GabrielMarquezMatte Nov 18, 2026
ca1931a
Merge branch 'fix/bughunt-writer' into develop
GabrielMarquezMatte Nov 18, 2026
251f564
fix(parser): skip indexers when FromAttributes reflects a model
GabrielMarquezMatte Nov 18, 2026
54b6f56
fix(reader): report string from SheetDataReader.GetFieldType for unre…
GabrielMarquezMatte Nov 18, 2026
3ba4b29
fix(parser): bind only the most-derived property when a model hides o…
GabrielMarquezMatte Nov 18, 2026
2042e25
Merge branch 'fix/bughunt-reader' into develop
GabrielMarquezMatte Nov 18, 2026
fbfa874
fix(native): zero every out param before validating arguments
GabrielMarquezMatte Nov 18, 2026
269cebd
fix(native): reject a NULL column-spec name as an invalid argument
GabrielMarquezMatte Nov 18, 2026
9b3e8ec
fix(python): copy native buffers without ctypes.string_at's C int size
GabrielMarquezMatte Nov 18, 2026
2fffc77
fix(native): let xl_open_memory take (NULL, 0) and always set a last …
GabrielMarquezMatte Nov 18, 2026
fd816f7
fix(python): reject an Arrow table that stays multi-chunk instead of …
GabrielMarquezMatte Nov 18, 2026
ceaab57
fix(parser): keep an ignored override from binding through its base d…
GabrielMarquezMatte Nov 18, 2026
b61b1d8
Merge branch 'fix/bughunt-parser' into develop
GabrielMarquezMatte Nov 18, 2026
0f75ecf
fix(rust): range-check only integer columns, bounded by the column le…
GabrielMarquezMatte Nov 18, 2026
0dbd60d
fix(generator): honor parser attributes inherited through overrides
GabrielMarquezMatte Nov 18, 2026
9faab39
fix(arrow): default ToArrowRecordBatch tolerates blanks past the infe…
GabrielMarquezMatte Nov 18, 2026
4c68b09
fix(cli): keep convert errors off the progress line and grow preview'…
GabrielMarquezMatte Nov 18, 2026
d0df8b9
Merge branch 'fix/bughunt-generator' into develop
GabrielMarquezMatte Nov 18, 2026
a42c214
fix(rust): give each aggregation failure its own status
GabrielMarquezMatte Nov 18, 2026
88f3348
Merge branch 'fix/bughunt-native' into develop
GabrielMarquezMatte Nov 18, 2026
d3b8ba7
Merge branch 'fix/bughunt-bindings' into develop
GabrielMarquezMatte Nov 18, 2026
584ac46
perf(reader): spool non-seekable inputs into one exact-size array
GabrielMarquezMatte Nov 18, 2026
dca3069
perf(reader): stop spooling a non-seekable input as soon as it passes…
GabrielMarquezMatte Nov 18, 2026
384285f
Merge branch 'perf/nonseekable-spool' into develop
GabrielMarquezMatte Nov 18, 2026
f286360
perf(typed): skip failing parses on blank cells
GabrielMarquezMatte Nov 18, 2026
9cdc871
Merge branch 'perf/blank-typed' into develop
GabrielMarquezMatte Nov 18, 2026
18a915d
perf(csv): yield parallel CSV rows without an async-iterator resume p…
GabrielMarquezMatte Nov 18, 2026
51d4002
Merge branch 'perf/csv-parallel-merge' into develop
GabrielMarquezMatte Nov 18, 2026
c4debbd
perf(zip): reuse the deflate hash table instead of clearing 256 KB pe…
GabrielMarquezMatte Nov 18, 2026
26cd12a
Merge branch 'perf/deflate-table' into develop
GabrielMarquezMatte Nov 18, 2026
35aace4
Merge branch 'master' into develop
GabrielMarquezMatte Nov 18, 2026
c994d37
fix(rust): satisfy clippy::err_expect in the new aggregation and rang…
GabrielMarquezMatte Nov 18, 2026
ab711ac
Merge branch 'develop' of https://github.com/GabrielMarquezMatte/Exce…
GabrielMarquezMatte Nov 18, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion python/src/excelreader/_native.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from collections.abc import Sequence
from functools import lru_cache
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any

if TYPE_CHECKING:
from excelreader.types import Border, CellStyle, OpenOptions, TableOptions, WriteOptions
Expand Down Expand Up @@ -84,6 +84,11 @@ def int32_len(buffer: bytes) -> int:
return length


def copy_bytes(pointer: Any, size: int) -> bytes:
"""Copies `size` bytes from native memory; `ctypes.string_at` takes a C int, so it truncates past 2 GiB."""
return (ctypes.c_char * size).from_address(ctypes.cast(pointer, ctypes.c_void_p).value).raw


class NativeColumnSpec(ctypes.Structure):
"""Mirrors xl_column_spec. `names`/`name_lens`/`name_count` may be left NULL/NULL/0 to resolve by
`index` instead. Build one with `column_spec_by_name`/`column_spec_by_names`, never directly —
Expand Down
8 changes: 4 additions & 4 deletions python/src/excelreader/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -615,20 +615,20 @@ def _buffer_to_array(raw: bytes, typecode: str, dtype: str) -> object:

def _decode_value_column(column: _native.NativeColumn, row_count: int) -> object:
element, typecode, dtype = _COLUMN_BUFFERS[ColumnType(column.type)]
raw = ctypes.string_at(column.values, row_count * ctypes.sizeof(element))
raw = _native.copy_bytes(column.values, row_count * ctypes.sizeof(element))
return _buffer_to_array(raw, typecode, dtype)


def _decode_string_column(column: _native.NativeColumn, row_count: int) -> StringColumn:
offsets_bytes = ctypes.string_at(column.values, (row_count + 1) * ctypes.sizeof(ctypes.c_int32))
data = ctypes.string_at(column.data, int(column.data_len)) if column.data_len else b""
offsets_bytes = _native.copy_bytes(column.values, (row_count + 1) * ctypes.sizeof(ctypes.c_int32))
data = _native.copy_bytes(column.data, int(column.data_len)) if column.data_len else b""
return StringColumn(_buffer_to_array(offsets_bytes, "i", "int32"), data)


def _decode_validity(column: _native.NativeColumn, row_count: int) -> bytes | None:
if not column.validity:
return None
return ctypes.string_at(column.validity, (row_count + 7) // 8)
return _native.copy_bytes(column.validity, (row_count + 7) // 8)


def _raw_options(
Expand Down
4 changes: 3 additions & 1 deletion python/src/excelreader/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,8 @@ def _table_from_arrow(batch: Any) -> tuple[TypedTable, list[ColumnType]]:

if isinstance(batch, pyarrow.Table):
batches = batch.combine_chunks().to_batches()
if len(batches) > 1:
raise ValueError("a column past 2 GiB stays split into chunks, which the int32-offset ABI cannot take")
batch = batches[0] if batches else pyarrow.RecordBatch.from_pylist([], schema=batch.schema)

types = [_arrow_column_type(field) for field in batch.schema]
Expand Down Expand Up @@ -433,7 +435,7 @@ def encrypt_package(
)
)
try:
encrypted = ctypes.string_at(buffer.data, buffer.len)
encrypted = _native.copy_bytes(buffer.data, buffer.len)
finally:
lib.xl_free_buffer(ctypes.byref(buffer))
if isinstance(destination, (str, os.PathLike)):
Expand Down
2 changes: 1 addition & 1 deletion python/src/excelreader/writer_handle.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ def _finished_bytes(self, handle: ctypes.c_void_p) -> bytes:
try:
if not buffer.data or buffer.len <= 0:
return b""
return ctypes.string_at(buffer.data, buffer.len)
return _native.copy_bytes(buffer.data, buffer.len)
finally:
self._lib.xl_free_buffer(ctypes.byref(buffer))

Expand Down
15 changes: 15 additions & 0 deletions python/tests/test_arrow_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,18 @@ def test_a_validity_mask_off_a_byte_boundary_converts(monkeypatch):
def test_an_unsupported_arrow_type_is_rejected():
with pytest.raises(ValueError, match="int8"):
_table_from_arrow(pa.record_batch({"small": pa.array([1], pa.int8())}))


def test_a_table_that_stays_multi_chunk_after_combining_is_rejected(monkeypatch):
batch = pa.record_batch({"n": pa.array([1, 2], pa.int64())})

class SplitTable:
def combine_chunks(self):
return self

def to_batches(self):
return [batch, batch]

monkeypatch.setattr(pa, "Table", SplitTable)
with pytest.raises(ValueError, match="2 GiB"):
_table_from_arrow(SplitTable())
8 changes: 8 additions & 0 deletions python/tests/test_native.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,3 +293,11 @@ def test_default_cell_style_marshals_every_color_as_unset():
assert raw.struct_size == 104
assert (raw.font_color, raw.fill_color) == (-1, -1)
assert [edge.color for edge in (raw.left, raw.right, raw.top, raw.bottom)] == [-1] * 4


def test_copy_bytes_reads_a_native_buffer_without_a_c_int_size_limit():
buffer = (ctypes.c_uint8 * 4)(1, 2, 3, 4)
pointer = ctypes.cast(buffer, ctypes.POINTER(ctypes.c_uint8))
assert _native.copy_bytes(pointer, 4) == b"\x01\x02\x03\x04"
assert _native.copy_bytes(ctypes.addressof(buffer), 2) == b"\x01\x02"
assert _native.copy_bytes(pointer, 0) == b""
30 changes: 30 additions & 0 deletions python/tests/test_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
)
from excelreader import _native
from excelreader import arrow_stream as arrow_stream_module
from excelreader import reader as reader_module

ENCRYPTED = Path(__file__).resolve().parents[2] / "tests" / "ExcelReader.Tests" / "data" / "encrypted"

Expand Down Expand Up @@ -856,3 +857,32 @@ def test_to_polars_reads_a_string_dict_column_across_batches(city_csv):
def test_parse_rejects_a_string_dict_column(city_csv):
with open_workbook(city_csv) as workbook, pytest.raises(ExcelReaderError, match="Arrow outputs"):
workbook.sheets[0].parse(_CITY_SCHEMA)


def test_column_decoding_does_not_go_through_the_int_sized_string_at(monkeypatch):
values = (ctypes.c_int64 * 3)(10, 20, 30)
offsets = (ctypes.c_int32 * 3)(0, 1, 3)
data = (ctypes.c_uint8 * 3)(97, 98, 99)
validity = (ctypes.c_uint8 * 1)(0b101)

def refuse(*_args):
raise AssertionError("ctypes.string_at takes a C int size")

monkeypatch.setattr(ctypes, "string_at", refuse)

ints = _native.NativeColumn(type=int(ColumnType.I64), length=3, values=ctypes.addressof(values))
assert list(reader_module._decode_value_column(ints, 3)) == [10, 20, 30]

strings = _native.NativeColumn(
type=int(ColumnType.STRING),
length=2,
values=ctypes.addressof(offsets),
data=ctypes.cast(data, ctypes.POINTER(ctypes.c_uint8)),
data_len=3,
)
decoded = reader_module._decode_string_column(strings, 2)
assert list(decoded.offsets) == [0, 1, 3]
assert decoded.data == b"abc"

ints.validity = ctypes.cast(validity, ctypes.POINTER(ctypes.c_uint8))
assert reader_module._decode_validity(ints, 3) == b"\x05"
138 changes: 101 additions & 37 deletions rust/excelreader/src/aggregate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,22 @@ use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
use std::path::{Path, PathBuf};
use std::sync::Mutex;

/// The status a shim returns to signal "a callback panicked". The library treats it like any other
/// caller code: from `accumulate` it may be discarded along with the partition instead of ending the
/// run, while from `seed` or `combine` it always ends the run, freeing every state and returning the
/// status verbatim. Either way, the wrapper then resumes the stored payload instead of turning it
/// into an [`Error`]. Positive, so it can never collide with a library status (every one of those is
/// `<= 0`) - but it CAN collide with a caller's own `Err(i32::MAX)`. `finish` does not disambiguate
/// the two by status; it keys on whether the panic slot actually holds a payload, so an ordinary
/// `Err(i32::MAX)` from `accumulate` or `combine` still surfaces as a normal [`Error`] rather than
/// being resumed as a panic.
/// The first of the positive statuses a shim returns for a callback failure. The library hands a
/// callback's non-zero status back verbatim, and a panic in `accumulate` can be discarded along with
/// its partition while the run goes on, so each failure gets its own status (`PANIC_STATUS - n`,
/// `n < RESERVED`) and `finish` maps the status the run ended on back to that exact failure. A
/// caller's own `Err` code inside that window is recorded the same way, so it never reads as a
/// panic; every library status is `<= 0`.
pub(crate) const PANIC_STATUS: i32 = i32::MAX;

/// How many distinct failure statuses one run hands out; a failure past this overwrites the last.
const RESERVED: usize = 1 << 16;

enum Failure {
Panic(Box<dyn Any + Send>),
Code(i32),
}

/// What each seeded accumulator is boxed in. The trailing byte keeps the allocation non-zero-sized,
/// so `seed` hands the library a distinct pointer per partition even when `A` is zero-sized, as the
/// header requires.
Expand Down Expand Up @@ -129,31 +134,48 @@ impl<'a, const N: usize> From<&'a [u8; N]> for CsvInput<'a> {

/// What every shim reaches through `user_data`. The C ABI shares one `user_data` across all worker
/// threads and requires it to be read-only or internally synchronized, so the seed closure is held
/// by shared reference and the panic slot behind a `Mutex` - the shims never take `&mut Shared`.
/// by shared reference and the failure list behind a `Mutex` - the shims never take `&mut Shared`.
struct Shared<'a, A, F> {
seed: &'a F,
panic: Mutex<Option<Box<dyn Any + Send>>>,
failures: Mutex<Vec<Failure>>,
_marker: std::marker::PhantomData<fn() -> A>,
}

impl<A, F> Shared<'_, A, F> {
/// Keeps the FIRST payload: later panics from sibling workers are dropped rather than racing to
/// overwrite the one that started the abort.
fn store_panic(&self, payload: Box<dyn Any + Send>) -> i32 {
let mut slot = self
.panic
fn record(&self, failure: Failure) -> i32 {
let mut failures = self
.failures
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if slot.is_none() {
*slot = Some(payload);
if failures.len() == RESERVED {
failures.pop();
}
PANIC_STATUS
failures.push(failure);
PANIC_STATUS - (failures.len() - 1) as i32
}

fn store_panic(&self, payload: Box<dyn Any + Send>) -> i32 {
self.record(Failure::Panic(payload))
}

/// Runs one user callback: its error code passes through and a panic is stored as above.
/// The failure `status` was handed out for, or `None` when it is not one of this run's.
fn failure(self, status: i32) -> Option<Failure> {
let index = usize::try_from(PANIC_STATUS.checked_sub(status)?).ok()?;
self.failures
.into_inner()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.into_iter()
.nth(index)
}

/// Runs one user callback: an error code passes through (one inside the failure window is
/// recorded so it cannot be mistaken for a panic) and a panic is stored as above.
fn guard(&self, body: impl FnOnce() -> Result<(), i32>) -> i32 {
match catch_unwind(AssertUnwindSafe(body)) {
Ok(Ok(())) => XL_OK,
Ok(Err(code)) if code > PANIC_STATUS - RESERVED as i32 => {
self.record(Failure::Code(code))
}
Ok(Err(code)) => code,
Err(payload) => self.store_panic(payload),
}
Expand Down Expand Up @@ -329,7 +351,7 @@ where
fn shared<A, F>(seed: &F) -> Shared<'_, A, F> {
Shared {
seed,
panic: Mutex::new(None),
failures: Mutex::new(Vec::new()),
_marker: std::marker::PhantomData,
}
}
Expand Down Expand Up @@ -415,16 +437,11 @@ where

/// Turns the run's outcome into a `Result`, taking ownership of the surviving state.
///
/// A stored panic only wins over the status if the status itself is [`PANIC_STATUS`]: an
/// `accumulate` panic on a chunk whose boundary guess was wrong is discarded exactly like a
/// non-panic error would be once the library re-reads that chunk from the confirmed start, so the
/// panic payload can be stale by the time this runs even though `shared.panic` still holds it. Only
/// resume it when the library's own verdict says the run actually ended on a panic; otherwise a
/// discarded panic from an abandoned first attempt would wrongly kill an otherwise-successful run
/// (or shadow the real error from an unrelated later failure). When the status does say the run
/// ended on a panic, the library has already unwound and freed every state it owns by the time it
/// returns, so resuming here delivers the original payload to the caller without it ever having
/// crossed a foreign frame.
/// The status the run ended on picks the failure to report. An `accumulate` panic on a chunk whose
/// boundary guess was wrong is discarded once the library re-reads that chunk, so its payload can
/// still be stored when the run ends; resuming only the failure the final status names keeps it from
/// killing an otherwise-successful run or shadowing the real outcome. A panic is resumed here, on the
/// calling thread, after the library has freed every state it owns.
fn finish<A, F>(shared: Shared<'_, A, F>, status: i32, state: *mut c_void) -> Result<A, Error>
where
F: Fn() -> A,
Expand All @@ -435,15 +452,13 @@ where
Some(unsafe { Box::from_raw(state as *mut State<A>) })
};

if status == PANIC_STATUS {
if let Some(payload) = shared
.panic
.into_inner()
.unwrap_or_else(|poisoned| poisoned.into_inner())
{
match shared.failure(status) {
Some(Failure::Panic(payload)) => {
drop(winner);
resume_unwind(payload);
}
Some(Failure::Code(code)) => return Err(status_error(code)),
None => {}
}
if status != XL_OK {
return Err(status_error(status));
Expand All @@ -469,3 +484,52 @@ pub(crate) fn status_error(status: i32) -> Error {
workbook::last_error(status)
}
}

#[cfg(test)]
mod tests {
use super::*;

fn payload_text(payload: Box<dyn Any + Send>) -> String {
payload.downcast::<&str>().map(|s| s.to_string()).unwrap()
}

#[test]
fn a_stale_panic_does_not_replace_a_later_one() {
let seed = || 0u8;
let shared = shared::<u8, _>(&seed);
shared.store_panic(Box::new("discarded"));
let status = shared.store_panic(Box::new("real"));

let payload = catch_unwind(AssertUnwindSafe(|| {
let _ = finish(shared, status, std::ptr::null_mut());
}))
.expect_err("the run ended on a panic");
assert_eq!(payload_text(payload), "real");
}

#[test]
fn a_stale_panic_does_not_shadow_a_callback_code() {
let seed = || 0u8;
let shared = shared::<u8, _>(&seed);
shared.store_panic(Box::new("discarded"));
let status = shared.guard(|| Err(PANIC_STATUS));

let error = finish(shared, status, std::ptr::null_mut())
.expect_err("the run ended on a callback code");
assert_eq!(error.kind(), ErrorKind::Callback(PANIC_STATUS));
}

#[test]
fn a_discarded_panic_does_not_fail_an_ok_run() {
let seed = || 0u8;
let shared = shared::<u8, _>(&seed);
shared.store_panic(Box::new("discarded"));
let state = Box::into_raw(Box::new(State {
value: 7u8,
_nonzero: 0,
}));

let value = finish(shared, XL_OK, state as *mut c_void).expect("the run succeeded");
assert_eq!(value, 7);
}
}
7 changes: 5 additions & 2 deletions rust/excelreader/src/workbook.rs
Original file line number Diff line number Diff line change
Expand Up @@ -862,13 +862,16 @@ fn out_of_range<T>(table: &XlTable, bindings: &[ColumnBinding<T>]) -> Option<Str
let Some(fits) = binding.fits else {
continue;
};
if col.r#type != ffi::XL_T_I64 {
continue;
}
let values = col.values as *const i64;
for row in 0..table.row_count {
for row in 0..table.row_count.min(col.length) {
let value = unsafe { *values.offset(row as isize) };
if is_valid(col, row) && !fits(value) {
return Some(format!(
"column '{}' holds {value} at row {row}, which its field's type cannot represent",
binding.names[0]
binding.names.first().copied().unwrap_or("")
));
}
}
Expand Down
Loading
Loading