diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index 67769c6f..d58739f4 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -12,7 +12,7 @@ from collections.abc import Sequence from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from excelreader.types import Border, CellStyle, OpenOptions, TableOptions, WriteOptions @@ -84,6 +84,11 @@ def int32_len(buffer: bytes) -> int: return length +def copy_bytes(pointer: Any, size: int) -> bytes: + """Copies `size` bytes from native memory; `ctypes.string_at` takes a C int, so it truncates past 2 GiB.""" + return (ctypes.c_char * size).from_address(ctypes.cast(pointer, ctypes.c_void_p).value).raw + + class NativeColumnSpec(ctypes.Structure): """Mirrors xl_column_spec. `names`/`name_lens`/`name_count` may be left NULL/NULL/0 to resolve by `index` instead. Build one with `column_spec_by_name`/`column_spec_by_names`, never directly — diff --git a/python/src/excelreader/reader.py b/python/src/excelreader/reader.py index eb990ddc..a204e1a3 100644 --- a/python/src/excelreader/reader.py +++ b/python/src/excelreader/reader.py @@ -615,20 +615,20 @@ def _buffer_to_array(raw: bytes, typecode: str, dtype: str) -> object: def _decode_value_column(column: _native.NativeColumn, row_count: int) -> object: element, typecode, dtype = _COLUMN_BUFFERS[ColumnType(column.type)] - raw = ctypes.string_at(column.values, row_count * ctypes.sizeof(element)) + raw = _native.copy_bytes(column.values, row_count * ctypes.sizeof(element)) return _buffer_to_array(raw, typecode, dtype) def _decode_string_column(column: _native.NativeColumn, row_count: int) -> StringColumn: - offsets_bytes = ctypes.string_at(column.values, (row_count + 1) * ctypes.sizeof(ctypes.c_int32)) - data = ctypes.string_at(column.data, int(column.data_len)) if column.data_len else b"" + offsets_bytes = _native.copy_bytes(column.values, (row_count + 1) * ctypes.sizeof(ctypes.c_int32)) + data = _native.copy_bytes(column.data, int(column.data_len)) if column.data_len else b"" return StringColumn(_buffer_to_array(offsets_bytes, "i", "int32"), data) def _decode_validity(column: _native.NativeColumn, row_count: int) -> bytes | None: if not column.validity: return None - return ctypes.string_at(column.validity, (row_count + 7) // 8) + return _native.copy_bytes(column.validity, (row_count + 7) // 8) def _raw_options( diff --git a/python/src/excelreader/writer.py b/python/src/excelreader/writer.py index 86bf4866..ddb56260 100644 --- a/python/src/excelreader/writer.py +++ b/python/src/excelreader/writer.py @@ -345,6 +345,8 @@ def _table_from_arrow(batch: Any) -> tuple[TypedTable, list[ColumnType]]: if isinstance(batch, pyarrow.Table): batches = batch.combine_chunks().to_batches() + if len(batches) > 1: + raise ValueError("a column past 2 GiB stays split into chunks, which the int32-offset ABI cannot take") batch = batches[0] if batches else pyarrow.RecordBatch.from_pylist([], schema=batch.schema) types = [_arrow_column_type(field) for field in batch.schema] @@ -433,7 +435,7 @@ def encrypt_package( ) ) try: - encrypted = ctypes.string_at(buffer.data, buffer.len) + encrypted = _native.copy_bytes(buffer.data, buffer.len) finally: lib.xl_free_buffer(ctypes.byref(buffer)) if isinstance(destination, (str, os.PathLike)): diff --git a/python/src/excelreader/writer_handle.py b/python/src/excelreader/writer_handle.py index 152016c3..3f52d8ea 100644 --- a/python/src/excelreader/writer_handle.py +++ b/python/src/excelreader/writer_handle.py @@ -221,7 +221,7 @@ def _finished_bytes(self, handle: ctypes.c_void_p) -> bytes: try: if not buffer.data or buffer.len <= 0: return b"" - return ctypes.string_at(buffer.data, buffer.len) + return _native.copy_bytes(buffer.data, buffer.len) finally: self._lib.xl_free_buffer(ctypes.byref(buffer)) diff --git a/python/tests/test_arrow_conversion.py b/python/tests/test_arrow_conversion.py index e8592eea..f12717a7 100644 --- a/python/tests/test_arrow_conversion.py +++ b/python/tests/test_arrow_conversion.py @@ -130,3 +130,18 @@ def test_a_validity_mask_off_a_byte_boundary_converts(monkeypatch): def test_an_unsupported_arrow_type_is_rejected(): with pytest.raises(ValueError, match="int8"): _table_from_arrow(pa.record_batch({"small": pa.array([1], pa.int8())})) + + +def test_a_table_that_stays_multi_chunk_after_combining_is_rejected(monkeypatch): + batch = pa.record_batch({"n": pa.array([1, 2], pa.int64())}) + + class SplitTable: + def combine_chunks(self): + return self + + def to_batches(self): + return [batch, batch] + + monkeypatch.setattr(pa, "Table", SplitTable) + with pytest.raises(ValueError, match="2 GiB"): + _table_from_arrow(SplitTable()) diff --git a/python/tests/test_native.py b/python/tests/test_native.py index 6db8f19e..e88b0f75 100644 --- a/python/tests/test_native.py +++ b/python/tests/test_native.py @@ -293,3 +293,11 @@ def test_default_cell_style_marshals_every_color_as_unset(): assert raw.struct_size == 104 assert (raw.font_color, raw.fill_color) == (-1, -1) assert [edge.color for edge in (raw.left, raw.right, raw.top, raw.bottom)] == [-1] * 4 + + +def test_copy_bytes_reads_a_native_buffer_without_a_c_int_size_limit(): + buffer = (ctypes.c_uint8 * 4)(1, 2, 3, 4) + pointer = ctypes.cast(buffer, ctypes.POINTER(ctypes.c_uint8)) + assert _native.copy_bytes(pointer, 4) == b"\x01\x02\x03\x04" + assert _native.copy_bytes(ctypes.addressof(buffer), 2) == b"\x01\x02" + assert _native.copy_bytes(pointer, 0) == b"" diff --git a/python/tests/test_reader.py b/python/tests/test_reader.py index 743e030a..b966b40e 100644 --- a/python/tests/test_reader.py +++ b/python/tests/test_reader.py @@ -20,6 +20,7 @@ ) from excelreader import _native from excelreader import arrow_stream as arrow_stream_module +from excelreader import reader as reader_module ENCRYPTED = Path(__file__).resolve().parents[2] / "tests" / "ExcelReader.Tests" / "data" / "encrypted" @@ -856,3 +857,32 @@ def test_to_polars_reads_a_string_dict_column_across_batches(city_csv): def test_parse_rejects_a_string_dict_column(city_csv): with open_workbook(city_csv) as workbook, pytest.raises(ExcelReaderError, match="Arrow outputs"): workbook.sheets[0].parse(_CITY_SCHEMA) + + +def test_column_decoding_does_not_go_through_the_int_sized_string_at(monkeypatch): + values = (ctypes.c_int64 * 3)(10, 20, 30) + offsets = (ctypes.c_int32 * 3)(0, 1, 3) + data = (ctypes.c_uint8 * 3)(97, 98, 99) + validity = (ctypes.c_uint8 * 1)(0b101) + + def refuse(*_args): + raise AssertionError("ctypes.string_at takes a C int size") + + monkeypatch.setattr(ctypes, "string_at", refuse) + + ints = _native.NativeColumn(type=int(ColumnType.I64), length=3, values=ctypes.addressof(values)) + assert list(reader_module._decode_value_column(ints, 3)) == [10, 20, 30] + + strings = _native.NativeColumn( + type=int(ColumnType.STRING), + length=2, + values=ctypes.addressof(offsets), + data=ctypes.cast(data, ctypes.POINTER(ctypes.c_uint8)), + data_len=3, + ) + decoded = reader_module._decode_string_column(strings, 2) + assert list(decoded.offsets) == [0, 1, 3] + assert decoded.data == b"abc" + + ints.validity = ctypes.cast(validity, ctypes.POINTER(ctypes.c_uint8)) + assert reader_module._decode_validity(ints, 3) == b"\x05" diff --git a/rust/excelreader/src/aggregate.rs b/rust/excelreader/src/aggregate.rs index 8ce0fc79..baabb4fe 100644 --- a/rust/excelreader/src/aggregate.rs +++ b/rust/excelreader/src/aggregate.rs @@ -16,17 +16,22 @@ use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; use std::path::{Path, PathBuf}; use std::sync::Mutex; -/// The status a shim returns to signal "a callback panicked". The library treats it like any other -/// caller code: from `accumulate` it may be discarded along with the partition instead of ending the -/// run, while from `seed` or `combine` it always ends the run, freeing every state and returning the -/// status verbatim. Either way, the wrapper then resumes the stored payload instead of turning it -/// into an [`Error`]. Positive, so it can never collide with a library status (every one of those is -/// `<= 0`) - but it CAN collide with a caller's own `Err(i32::MAX)`. `finish` does not disambiguate -/// the two by status; it keys on whether the panic slot actually holds a payload, so an ordinary -/// `Err(i32::MAX)` from `accumulate` or `combine` still surfaces as a normal [`Error`] rather than -/// being resumed as a panic. +/// The first of the positive statuses a shim returns for a callback failure. The library hands a +/// callback's non-zero status back verbatim, and a panic in `accumulate` can be discarded along with +/// its partition while the run goes on, so each failure gets its own status (`PANIC_STATUS - n`, +/// `n < RESERVED`) and `finish` maps the status the run ended on back to that exact failure. A +/// caller's own `Err` code inside that window is recorded the same way, so it never reads as a +/// panic; every library status is `<= 0`. pub(crate) const PANIC_STATUS: i32 = i32::MAX; +/// How many distinct failure statuses one run hands out; a failure past this overwrites the last. +const RESERVED: usize = 1 << 16; + +enum Failure { + Panic(Box), + Code(i32), +} + /// What each seeded accumulator is boxed in. The trailing byte keeps the allocation non-zero-sized, /// so `seed` hands the library a distinct pointer per partition even when `A` is zero-sized, as the /// header requires. @@ -129,31 +134,48 @@ impl<'a, const N: usize> From<&'a [u8; N]> for CsvInput<'a> { /// What every shim reaches through `user_data`. The C ABI shares one `user_data` across all worker /// threads and requires it to be read-only or internally synchronized, so the seed closure is held -/// by shared reference and the panic slot behind a `Mutex` - the shims never take `&mut Shared`. +/// by shared reference and the failure list behind a `Mutex` - the shims never take `&mut Shared`. struct Shared<'a, A, F> { seed: &'a F, - panic: Mutex>>, + failures: Mutex>, _marker: std::marker::PhantomData A>, } impl Shared<'_, A, F> { - /// Keeps the FIRST payload: later panics from sibling workers are dropped rather than racing to - /// overwrite the one that started the abort. - fn store_panic(&self, payload: Box) -> i32 { - let mut slot = self - .panic + fn record(&self, failure: Failure) -> i32 { + let mut failures = self + .failures .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - if slot.is_none() { - *slot = Some(payload); + if failures.len() == RESERVED { + failures.pop(); } - PANIC_STATUS + failures.push(failure); + PANIC_STATUS - (failures.len() - 1) as i32 + } + + fn store_panic(&self, payload: Box) -> i32 { + self.record(Failure::Panic(payload)) } - /// Runs one user callback: its error code passes through and a panic is stored as above. + /// The failure `status` was handed out for, or `None` when it is not one of this run's. + fn failure(self, status: i32) -> Option { + let index = usize::try_from(PANIC_STATUS.checked_sub(status)?).ok()?; + self.failures + .into_inner() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .into_iter() + .nth(index) + } + + /// Runs one user callback: an error code passes through (one inside the failure window is + /// recorded so it cannot be mistaken for a panic) and a panic is stored as above. fn guard(&self, body: impl FnOnce() -> Result<(), i32>) -> i32 { match catch_unwind(AssertUnwindSafe(body)) { Ok(Ok(())) => XL_OK, + Ok(Err(code)) if code > PANIC_STATUS - RESERVED as i32 => { + self.record(Failure::Code(code)) + } Ok(Err(code)) => code, Err(payload) => self.store_panic(payload), } @@ -329,7 +351,7 @@ where fn shared(seed: &F) -> Shared<'_, A, F> { Shared { seed, - panic: Mutex::new(None), + failures: Mutex::new(Vec::new()), _marker: std::marker::PhantomData, } } @@ -415,16 +437,11 @@ where /// Turns the run's outcome into a `Result`, taking ownership of the surviving state. /// -/// A stored panic only wins over the status if the status itself is [`PANIC_STATUS`]: an -/// `accumulate` panic on a chunk whose boundary guess was wrong is discarded exactly like a -/// non-panic error would be once the library re-reads that chunk from the confirmed start, so the -/// panic payload can be stale by the time this runs even though `shared.panic` still holds it. Only -/// resume it when the library's own verdict says the run actually ended on a panic; otherwise a -/// discarded panic from an abandoned first attempt would wrongly kill an otherwise-successful run -/// (or shadow the real error from an unrelated later failure). When the status does say the run -/// ended on a panic, the library has already unwound and freed every state it owns by the time it -/// returns, so resuming here delivers the original payload to the caller without it ever having -/// crossed a foreign frame. +/// The status the run ended on picks the failure to report. An `accumulate` panic on a chunk whose +/// boundary guess was wrong is discarded once the library re-reads that chunk, so its payload can +/// still be stored when the run ends; resuming only the failure the final status names keeps it from +/// killing an otherwise-successful run or shadowing the real outcome. A panic is resumed here, on the +/// calling thread, after the library has freed every state it owns. fn finish(shared: Shared<'_, A, F>, status: i32, state: *mut c_void) -> Result where F: Fn() -> A, @@ -435,15 +452,13 @@ where Some(unsafe { Box::from_raw(state as *mut State) }) }; - if status == PANIC_STATUS { - if let Some(payload) = shared - .panic - .into_inner() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - { + match shared.failure(status) { + Some(Failure::Panic(payload)) => { drop(winner); resume_unwind(payload); } + Some(Failure::Code(code)) => return Err(status_error(code)), + None => {} } if status != XL_OK { return Err(status_error(status)); @@ -469,3 +484,52 @@ pub(crate) fn status_error(status: i32) -> Error { workbook::last_error(status) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn payload_text(payload: Box) -> String { + payload.downcast::<&str>().map(|s| s.to_string()).unwrap() + } + + #[test] + fn a_stale_panic_does_not_replace_a_later_one() { + let seed = || 0u8; + let shared = shared::(&seed); + shared.store_panic(Box::new("discarded")); + let status = shared.store_panic(Box::new("real")); + + let payload = catch_unwind(AssertUnwindSafe(|| { + let _ = finish(shared, status, std::ptr::null_mut()); + })) + .expect_err("the run ended on a panic"); + assert_eq!(payload_text(payload), "real"); + } + + #[test] + fn a_stale_panic_does_not_shadow_a_callback_code() { + let seed = || 0u8; + let shared = shared::(&seed); + shared.store_panic(Box::new("discarded")); + let status = shared.guard(|| Err(PANIC_STATUS)); + + let error = finish(shared, status, std::ptr::null_mut()) + .expect_err("the run ended on a callback code"); + assert_eq!(error.kind(), ErrorKind::Callback(PANIC_STATUS)); + } + + #[test] + fn a_discarded_panic_does_not_fail_an_ok_run() { + let seed = || 0u8; + let shared = shared::(&seed); + shared.store_panic(Box::new("discarded")); + let state = Box::into_raw(Box::new(State { + value: 7u8, + _nonzero: 0, + })); + + let value = finish(shared, XL_OK, state as *mut c_void).expect("the run succeeded"); + assert_eq!(value, 7); + } +} diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 13a8f201..63ca79e5 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -862,13 +862,16 @@ fn out_of_range(table: &XlTable, bindings: &[ColumnBinding]) -> Option Vec> { + use excelreader::__private::{column_bool, column_i64, ColumnBinding}; + vec![ + ColumnBinding { + names: &[], + column_type: ColumnType::I64, + assign: |row: &mut HandWritten, col, i| row.value = column_i64(col, i), + fits: Some(|value| value < 100), + }, + ColumnBinding { + names: &["flag"], + column_type: ColumnType::Bool, + assign: |row: &mut HandWritten, col, i| row.flag = column_bool(col, i), + fits: Some(|_| false), + }, + ] + } +} + +#[test] +fn a_range_check_on_a_non_integer_column_is_ignored() { + let path = write_csv("hand-written", "n,flag\n1,true\n2,false\n"); + let workbook = OpenOptions::new() + .format(Format::Csv) + .open(&path) + .expect("open must succeed"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("a fits check on a bool column must not read it as i64"); + assert_eq!(table.len(), 2); + drop(workbook); + let _ = std::fs::remove_file(path); +} + +#[test] +fn an_out_of_range_value_in_an_unnamed_binding_reports_instead_of_panicking() { + let path = write_csv("hand-written-range", "n,flag\n300,true\n"); + let workbook = OpenOptions::new() + .format(Format::Csv) + .open(&path) + .expect("open must succeed"); + let Err(error) = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + else { + panic!("300 does not fit"); + }; + assert_eq!(error.kind(), excelreader::ErrorKind::InvalidArgument); + drop(workbook); + let _ = std::fs::remove_file(path); +} diff --git a/src/ExcelReader.Arrow/ArrowConversionExtensions.cs b/src/ExcelReader.Arrow/ArrowConversionExtensions.cs index c520ec74..82976c44 100644 --- a/src/ExcelReader.Arrow/ArrowConversionExtensions.cs +++ b/src/ExcelReader.Arrow/ArrowConversionExtensions.cs @@ -26,7 +26,7 @@ public static RecordBatch ToArrowRecordBatch(this IExcelSheet sheet, ArrowReadOp options ??= ArrowReadOptions.Default; ArgumentOutOfRangeException.ThrowIfNegative(options.DegreeOfParallelism, nameof(options)); int headerRow = options.HeaderRow; - IReadOnlyList schema = options.Schema ?? Excel.InferSchema(sheet, new SchemaInferenceOptions { HeaderRow = headerRow }); + IReadOnlyList schema = options.Schema ?? InferNullableSchema(sheet, headerRow); TypedColumnSpec[] specs = ToSpecs(schema); using ColumnarTable table = TypedTableParser.Parse(sheet, specs, headerRow, options.DegreeOfParallelism, allowDictionary: true, chunkSizeOverride: 0, out _, out string? specError) ?? throw new ArgumentException(specError, nameof(options)); @@ -41,6 +41,17 @@ public static RecordBatch ToArrowRecordBatch(this IExcelSheet sheet, ArrowReadOp return new RecordBatch(new Schema(fields, metadata: null), arrays, checked((int)table.RowCount)); } + private static ExcelColumnSchema[] InferNullableSchema(IExcelSheet sheet, int headerRow) + { + IReadOnlyList inferred = Excel.InferSchema(sheet, new SchemaInferenceOptions { HeaderRow = headerRow }); + ExcelColumnSchema[] schema = new ExcelColumnSchema[inferred.Count]; + for (int i = 0; i < schema.Length; i++) + { + schema[i] = inferred[i] with { IsNullable = true }; + } + return schema; + } + private static TypedColumnSpec[] ToSpecs(IReadOnlyList schema) { TypedColumnSpec[] specs = new TypedColumnSpec[schema.Count]; diff --git a/src/ExcelReader.Cli/CliCommands.cs b/src/ExcelReader.Cli/CliCommands.cs index ddc8e29c..9ce455ec 100644 --- a/src/ExcelReader.Cli/CliCommands.cs +++ b/src/ExcelReader.Cli/CliCommands.cs @@ -34,21 +34,35 @@ internal static int Sheets(string path, TextWriter stdout, TextWriter stderr, st internal static int Convert(string path, string? sheet, string? output, string? format, char delimiter, Stream stdout, TextWriter stderr, Action? onProgress = null, string? password = null, char? inputDelimiter = null) { + bool progressPending = false; + Action? progress = onProgress is null ? null : rows => + { + progressPending = true; + onProgress(rows); + }; return Execute(() => { - string resolvedFormat = ResolveFormat(format, output); - ThrowIfOutputIsADirectory(output); - using IExcelWorkbook workbook = Open(path, sheet, out IExcelSheet selected, password, inputDelimiter); - - if (output is null) + try { - WriteFormat(resolvedFormat, selected, stdout, leaveOpen: true, delimiter, onProgress); + string resolvedFormat = ResolveFormat(format, output); + ThrowIfOutputIsADirectory(output); + using IExcelWorkbook workbook = Open(path, sheet, out IExcelSheet selected, password, inputDelimiter); + + if (output is null) + { + WriteFormat(resolvedFormat, selected, stdout, leaveOpen: true, delimiter, progress); + return 0; + } + using StagedFile staged = new(output); + WriteFormat(resolvedFormat, selected, staged.Stream, leaveOpen: true, delimiter, progress); + staged.Commit(); return 0; } - using StagedFile staged = new(output); - WriteFormat(resolvedFormat, selected, staged.Stream, leaveOpen: true, delimiter, onProgress); - staged.Commit(); - return 0; + catch when (progressPending) + { + stderr.WriteLine(); + throw; + } }, stderr); } @@ -245,7 +259,7 @@ internal static int Preview(string path, string? sheet, int headerRow, int rows, int headCount = fast ? rows : (rows + 1) / 2; int tailCount = rows - headCount; List head = []; - string?[][] tail = new string?[tailCount][]; + List tail = []; long total = 0; int columnCount = schema.Count; using IExcelRowEnumerator enumerator = selected.GetEnumerator(); @@ -264,7 +278,16 @@ internal static int Preview(string path, string? sheet, int headerRow, int rows, } if (total >= headCount && tailCount > 0) { - tail[(total - headCount) % tailCount] = Materialize(row, selected.IsDate1904, schema, filter); + string?[] cells = Materialize(row, selected.IsDate1904, schema, filter); + int slot = (int)((total - headCount) % tailCount); + if (slot < tail.Count) + { + tail[slot] = cells; + } + if (slot == tail.Count) + { + tail.Add(cells); + } } total++; if (fast && total > rows) @@ -282,7 +305,7 @@ internal static int Preview(string path, string? sheet, int headerRow, int rows, } for (long j = tailRegion - kept; j < tailRegion; j++) { - body.Add(tail[j % tailCount]); + body.Add(tail[(int)(j % tailCount)]); } string shownTotal = fast && total > rows ? "?" : Underscored(total); WriteTable(stdout, schema, shownTotal, selection ?? [.. Enumerable.Range(0, columnCount)], body, maxColumns, maxWidth); diff --git a/src/ExcelReader.Cli/Commands.cs b/src/ExcelReader.Cli/Commands.cs index c39b0193..69695b5e 100644 --- a/src/ExcelReader.Cli/Commands.cs +++ b/src/ExcelReader.Cli/Commands.cs @@ -34,7 +34,10 @@ public int Convert([Argument] string path, string? sheet = null, string? output // ponytail: progress is a carriage-returned line on stderr rather than a spinner widget. int code = CliCommands.Convert(path, sheet, output, format, delimiter ?? ',', stdout, stderr, rowsWritten => Console.Error.Write($"\rConverting... {rowsWritten.ToString("N0", CultureInfo.InvariantCulture)} rows written"), password, inputDelimiter); - Console.Error.WriteLine(); + if (code == 0) + { + Console.Error.WriteLine(); + } return code; } diff --git a/src/ExcelReader.Core/Parser/CsvEnumerable.cs b/src/ExcelReader.Core/Parser/CsvEnumerable.cs index 71e0a86f..a7f49d4f 100644 --- a/src/ExcelReader.Core/Parser/CsvEnumerable.cs +++ b/src/ExcelReader.Core/Parser/CsvEnumerable.cs @@ -25,15 +25,13 @@ public sealed class CsvEnumerable : IEnumerable, IAsyncEnumerable private readonly ExcelParserOptions _config; private readonly CancellationToken _ct; private readonly TypeMapInfo _info; - private readonly CsvReader? _ownedReader; - internal CsvEnumerable(CsvSheet sheet, ExcelParserOptions config, TypeMapInfo info, CsvReader? ownedReader, CancellationToken ct) + internal CsvEnumerable(CsvSheet sheet, ExcelParserOptions config, TypeMapInfo info, CancellationToken ct) { _sheet = sheet; _config = config; _info = info; _ct = ct; - _ownedReader = ownedReader; } internal CsvEnumerable(CsvSheet sheet, ExcelParserOptions config, TypeMapInfo explicitInfo) @@ -68,8 +66,18 @@ IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken can /// public AsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { - CancellationToken effective = cancellationToken.CanBeCanceled ? cancellationToken : _ct; - return new AsyncEnumerator(_sheet, _info, _config.ColumnNameComparer, _config.HeaderNormalization, _config.HeaderRow, _config.Culture, _config.ThrowOnParseFailure, _ownedReader, effective); + if (!cancellationToken.CanBeCanceled || !_ct.CanBeCanceled) + { + CancellationToken effective = cancellationToken.CanBeCanceled ? cancellationToken : _ct; + return CreateAsyncEnumerator(linked: null, effective); + } + var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _ct); + return CreateAsyncEnumerator(linked, linked.Token); + } + + private AsyncEnumerator CreateAsyncEnumerator(CancellationTokenSource? linked, CancellationToken ct) + { + return new AsyncEnumerator(_sheet, _info, _config.ColumnNameComparer, _config.HeaderNormalization, _config.HeaderRow, _config.Culture, _config.ThrowOnParseFailure, linked, ct); } /// Enumerates CSV rows synchronously, projecting each into a instance by fixed field index. @@ -107,7 +115,7 @@ private protected override T Project() public sealed class AsyncEnumerator : AsyncRowEnumerator { private CsvRowProjector _projector; - private readonly CsvReader? _ownedReader; + private readonly CancellationTokenSource? _linked; internal AsyncEnumerator( CsvSheet sheet, @@ -117,21 +125,24 @@ internal AsyncEnumerator( int headerRow, IFormatProvider provider, bool throwOnParseFailure, - CsvReader? ownedReader, + CancellationTokenSource? linked, CancellationToken ct) : base(sheet, ct) { _projector = new CsvRowProjector(typeInfo, comparer, normalization, headerRow, provider, throwOnParseFailure); - _ownedReader = ownedReader; + _linked = linked; } /// public override async ValueTask DisposeAsync() { - await base.DisposeAsync().ConfigureAwait(false); - if (_ownedReader is not null) + try + { + await base.DisposeAsync().ConfigureAwait(false); + } + finally { - await _ownedReader.DisposeAsync().ConfigureAwait(false); + _linked?.Dispose(); } } diff --git a/src/ExcelReader.Core/Parser/ExcelEnumerable.cs b/src/ExcelReader.Core/Parser/ExcelEnumerable.cs index 6937b258..e5295ef7 100644 --- a/src/ExcelReader.Core/Parser/ExcelEnumerable.cs +++ b/src/ExcelReader.Core/Parser/ExcelEnumerable.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using ExcelReader.Core.Parser.Internal; using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Csv; using ExcelReader.Core.Reader.Xlsx; namespace ExcelReader.Core.Parser @@ -90,7 +91,7 @@ internal Enumerator( bool throwOnParseFailure = false) : base(rows) { - _projector = new RowProjector(typeInfo, comparer, normalization, headerRow, isDate1904, provider, throwOnParseFailure); + _projector = new RowProjector(typeInfo, comparer, normalization, headerRow, isDate1904, provider, throwOnParseFailure, rows is CsvReader.Enumerator); } private protected override ProjectionStep Classify() @@ -122,7 +123,7 @@ internal AsyncEnumerator( CancellationToken ct) : base(sheet, ct) { - _projector = new RowProjector(typeInfo, comparer, normalization, headerRow, sheet.IsDate1904, provider, throwOnParseFailure); + _projector = new RowProjector(typeInfo, comparer, normalization, headerRow, sheet.IsDate1904, provider, throwOnParseFailure, sheet is CsvSheet); } private protected override ProjectionStep Classify() diff --git a/src/ExcelReader.Core/Parser/ExcelParser.cs b/src/ExcelReader.Core/Parser/ExcelParser.cs index 1abed23c..e713131f 100644 --- a/src/ExcelReader.Core/Parser/ExcelParser.cs +++ b/src/ExcelReader.Core/Parser/ExcelParser.cs @@ -74,6 +74,10 @@ public ExcelEnumerable Parse(IExcelSheet sh { ArgumentNullException.ThrowIfNull(sheet); ExcelTableSheet.ThrowIfHeaderless(sheet); + if (sheet is CsvSheet) + { + return new ExcelEnumerable(sheet, _config, _csvInfo()); + } return new ExcelEnumerable(sheet, _config, _info()); } @@ -83,7 +87,7 @@ public ExcelEnumerable Parse(IExcelSheet sh /// /// Uses a specialized enumerable (dense field binding, single-pass projection), since CSV rows have /// no gaps or styles. Prefer this overload over for CSV: a sheet - /// held as goes through the generic path. + /// held as binds the same way but through the slower generic path. /// public CsvEnumerable Parse(CsvSheet sheet) { diff --git a/src/ExcelReader.Core/Parser/ExcelParserOptions.cs b/src/ExcelReader.Core/Parser/ExcelParserOptions.cs index 6b96934e..5228beb0 100644 --- a/src/ExcelReader.Core/Parser/ExcelParserOptions.cs +++ b/src/ExcelReader.Core/Parser/ExcelParserOptions.cs @@ -18,8 +18,9 @@ public sealed class ExcelParserOptions public HeaderNormalization HeaderNormalization { get; init; } = HeaderNormalization.Trim; /// - /// Gets the culture used when parsing text-backed numeric and cells (e.g. pt-BR - /// "1.234,56"). Binary numeric cells from XLS/XLSB carry a raw double and ignore this setting. + /// Gets the culture used when parsing text cells (CSV fields, XLSX/XLSB/XLS strings) as numbers or + /// (e.g. pt-BR "1.234,56"). Numeric and date cells stored by the workbook are + /// always read invariantly and ignore this setting. /// Defaults to . /// public CultureInfo Culture { get; init; } = CultureInfo.InvariantCulture; diff --git a/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs b/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs index 5f0c926b..fb27f36d 100644 --- a/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs +++ b/src/ExcelReader.Core/Parser/Internal/ColumnParserFactory.cs @@ -309,6 +309,7 @@ internal static bool ReadTextTimeOnly(in Cell cell, in ExcelRowContext context, internal static bool TryParseSpanParsable(in Cell cell, IFormatProvider provider, [MaybeNullWhen(false)] out TValue value) where TValue : ISpanParsable { + provider = cell.TextProvider(provider); // XLSB and XLS keep a number in binary, with no text in Value to parse. if (cell.Value.IsEmpty && cell.Type is CellType.Number or CellType.Date) { diff --git a/src/ExcelReader.Core/Parser/Internal/MappableProperties.cs b/src/ExcelReader.Core/Parser/Internal/MappableProperties.cs new file mode 100644 index 00000000..1064283a --- /dev/null +++ b/src/ExcelReader.Core/Parser/Internal/MappableProperties.cs @@ -0,0 +1,27 @@ +using System.Diagnostics.CodeAnalysis; +using System.Reflection; + +namespace ExcelReader.Core.Parser.Internal +{ + internal static class MappableProperties + { + // Most-derived type first, so a property hidden with `new` loses to the one hiding it, as in the source generator. + [RequiresUnreferencedCode("Reflects over the type's public properties, which trimming may remove.")] + internal static List Of(Type type) + { + var seenNames = new HashSet(StringComparer.Ordinal); + var properties = new List(); + for (Type? current = type; current is not null; current = current.BaseType) + { + foreach (PropertyInfo prop in current.GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + { + if (prop.GetIndexParameters().Length == 0 && seenNames.Add(prop.Name) && !Attribute.IsDefined(prop, typeof(ExcelIgnoreAttribute))) + { + properties.Add(prop); + } + } + } + return properties; + } + } +} diff --git a/src/ExcelReader.Core/Parser/Internal/RowProjector.cs b/src/ExcelReader.Core/Parser/Internal/RowProjector.cs index 19ab1294..d04371ad 100644 --- a/src/ExcelReader.Core/Parser/Internal/RowProjector.cs +++ b/src/ExcelReader.Core/Parser/Internal/RowProjector.cs @@ -13,12 +13,13 @@ internal struct RowProjector private readonly int _headerRow; private readonly ExcelRowContext _context; private readonly bool _throwOnParseFailure; + private readonly bool _skipBlankRows; private ColumnBinding[]? _bindings; private bool[] _seen; private int _requireValueCount; private int _rowNumber; - internal RowProjector(TypeMapInfo typeInfo, StringComparer comparer, HeaderNormalization normalization, int headerRow, bool isDate1904, IFormatProvider provider, bool throwOnParseFailure = false) + internal RowProjector(TypeMapInfo typeInfo, StringComparer comparer, HeaderNormalization normalization, int headerRow, bool isDate1904, IFormatProvider provider, bool throwOnParseFailure = false, bool skipBlankRows = false) { _typeInfo = typeInfo; _comparer = comparer; @@ -26,6 +27,7 @@ internal RowProjector(TypeMapInfo typeInfo, StringComparer comparer, HeaderNo _headerRow = headerRow; _context = new ExcelRowContext { IsDate1904 = isDate1904, FormatProvider = provider }; _throwOnParseFailure = throwOnParseFailure; + _skipBlankRows = skipBlankRows; _seen = []; } @@ -38,9 +40,13 @@ internal ProjectionStep Classify(in Row row) BuildIndexColumnMap(); } _rowNumber++; - return ProjectionStep.Yield; + return YieldUnlessBlank(in row); } ProjectionStep step = ProjectionRules.ClassifyRow(ref _rowNumber, _headerRow, _bindings is not null); + if (step == ProjectionStep.Yield) + { + return YieldUnlessBlank(in row); + } if (step != ProjectionStep.BuildMap) { return step; @@ -49,6 +55,16 @@ internal ProjectionStep Classify(in Row row) return ProjectionStep.Skip; } + // A CSV blank line is a record of one empty field; the CSV-specialized projector skips it the same way. + private readonly ProjectionStep YieldUnlessBlank(in Row row) + { + if (_skipBlankRows && row.Descriptors is [{ Type: CellType.Empty }]) + { + return ProjectionStep.Skip; + } + return ProjectionStep.Yield; + } + [MethodImpl(MethodImplOptions.NoInlining)] internal readonly T Project(Row row) { diff --git a/src/ExcelReader.Core/Parser/Internal/TypeMapper.cs b/src/ExcelReader.Core/Parser/Internal/TypeMapper.cs index 2fb0bbf6..4f15224e 100644 --- a/src/ExcelReader.Core/Parser/Internal/TypeMapper.cs +++ b/src/ExcelReader.Core/Parser/Internal/TypeMapper.cs @@ -33,7 +33,7 @@ private static TypeMapInfo BuildCsvInfo() private static bool HasDateProperty() { - return typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance).Any(static prop => + return MappableProperties.Of(typeof(T)).Any(static prop => { Type effective = Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType; return effective == typeof(DateTime) || effective == typeof(DateOnly) || effective == typeof(TimeOnly); @@ -42,17 +42,12 @@ private static bool HasDateProperty() private static TypeMapInfo Build(bool csvTextDates) { - PropertyInfo[] properties = typeof(T) - .GetProperties(BindingFlags.Public | BindingFlags.Instance); + List properties = MappableProperties.Of(typeof(T)); - var propertyMaps = new List>(properties.Length); + var propertyMaps = new List>(properties.Count); foreach (PropertyInfo prop in properties) { - if (Attribute.IsDefined(prop, typeof(ExcelIgnoreAttribute))) - { - continue; - } ExcelRequiredAttribute? requiredAttr = prop.GetCustomAttribute(); bool isRequired = requiredAttr is not null; if (prop.GetSetMethod() is null) diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/CsvHeaderBinder.cs b/src/ExcelReader.Core/Parser/ParallelCsv/CsvHeaderBinder.cs index d6a205d7..c591c87d 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/CsvHeaderBinder.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/CsvHeaderBinder.cs @@ -30,8 +30,8 @@ internal static CsvBoundColumnMap Bind( firstDataRecordOffset = rows.MoveNext() ? rows.CurrentRecordStart : long.MaxValue; return map; } - throw new InvalidOperationException( - $"The CSV source has no row at header index {config.HeaderRow}."); + firstDataRecordOffset = long.MaxValue; + return new CsvBoundColumnMap([], [], [], []); } } } diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs index bd4eebe3..8f505b2f 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs @@ -19,6 +19,7 @@ internal sealed class ParallelCsvEnumerable : IAsyncEnumerable private readonly ReaderLifetime? _lease; private readonly object? _owner; + private readonly CancellationToken _ct; internal ParallelCsvEnumerable( CsvChunkSource source, @@ -30,7 +31,8 @@ internal ParallelCsvEnumerable( ExcelParserOptions config, int degreeOfParallelism, ReaderLifetime? lease = null, - object? owner = null) + object? owner = null, + CancellationToken ct = default) { _source = source; _plan = plan; @@ -42,59 +44,12 @@ internal ParallelCsvEnumerable( _dop = degreeOfParallelism; _lease = lease; _owner = owner; + _ct = ct; } - [SuppressMessage("Reliability", "CA2025:Do not pass 'IDisposable' instances into unawaited tasks", - Justification = "The CTS and semaphore handed to the workers outlive them by construction: the finally block cancels and then awaits every worker task to completion, and only the enclosing `using` declarations' finally — which runs after it — disposes them.")] - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) { - _lease?.Acquire(_owner!); - using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - CancellationToken ct = cts.Token; - - int inFlight = _dop; - using var slots = new SemaphoreSlim(inFlight, inFlight); - - int ring = inFlight + 1; - - var lists = new ConcurrentBag>(); - MergeState state = StartWorkers(slots, lists, ring, ct); - Task allWorkers = Task.WhenAll(state.Workers); - long recordsBefore = 0; - - try - { - long confirmedNextStart = _firstDataRecordOffset; - for (int i = 0; i < _plan.Count; i++) - { - ct.ThrowIfCancellationRequested(); - CsvChunkResult result = await TakeChunkAsync( - state, slots, i, ring, confirmedNextStart, allWorkers, ct).ConfigureAwait(false); - - List models = result.Models; - int count = models.Count; - for (int m = 0; m < count; m++) - { - yield return models[m]; - } - - models.Clear(); - lists.Add(models); - - confirmedNextStart = AdvanceOrThrow(result, recordsBefore); - recordsBefore += result.Records; - if (confirmedNextStart >= long.MaxValue) - { - break; - } - } - } - finally - { - await cts.CancelAsync().ConfigureAwait(false); - await allWorkers.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); - _lease?.Release(); - } + return new Enumerator(this, cancellationToken); } private long AdvanceOrThrow(CsvChunkResult result, long recordsBefore) @@ -220,5 +175,133 @@ private static async Task WaitForChunkAsync(Task chunkReady, Task allWorke } await chunkReady.ConfigureAwait(false); } + + private sealed class Enumerator : IAsyncEnumerator + { + private readonly ParallelCsvEnumerable _parent; + private readonly CancellationToken _cancellationToken; + private readonly ConcurrentBag> _lists = new(); + private CancellationTokenSource? _cts; + private SemaphoreSlim? _slots; + private CancellationToken _ct; + private MergeState _state; + private Task _allWorkers = Task.CompletedTask; + private int _ring; + private int _next; + private long _confirmedNextStart; + private long _recordsBefore; + private CsvChunkResult? _chunk; + private List? _list; + private int _index; + private int _count; + private bool _done; + + internal Enumerator(ParallelCsvEnumerable parent, CancellationToken cancellationToken) + { + _parent = parent; + _cancellationToken = cancellationToken; + _confirmedNextStart = parent._firstDataRecordOffset; + } + + public T Current { get; private set; } = default!; + + public ValueTask MoveNextAsync() + { + if (_list is not null && _index < _count) + { + Current = _list[_index++]; + return new ValueTask(true); + } + return NextChunkAsync(); + } + + private async ValueTask NextChunkAsync() + { + if (_done) + { + return false; + } + try + { + if (_cts is null) + { + Start(); + } + while (true) + { + if ((_chunk is not null && !FinishChunk()) || _next >= _parent._plan.Count) + { + await DisposeAsync().ConfigureAwait(false); + return false; + } + _ct.ThrowIfCancellationRequested(); + CsvChunkResult result = await _parent.TakeChunkAsync( + _state, _slots!, _next, _ring, _confirmedNextStart, _allWorkers, _ct).ConfigureAwait(false); + _next++; + _chunk = result; + _list = result.Models; + _count = _list.Count; + _index = 0; + if (_count > 0) + { + Current = _list[_index++]; + return true; + } + } + } + catch + { + await DisposeAsync().ConfigureAwait(false); + throw; + } + } + + [SuppressMessage("Reliability", "CA2025:Do not pass 'IDisposable' instances into unawaited tasks", + Justification = "The CTS and semaphore handed to the workers outlive them by construction: DisposeAsync cancels and then awaits every worker task to completion before disposing them.")] + private void Start() + { + _parent._lease?.Acquire(_parent._owner!); + _cts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken, _parent._ct); + _ct = _cts.Token; + _slots = new SemaphoreSlim(_parent._dop, _parent._dop); + _ring = _parent._dop + 1; + _state = _parent.StartWorkers(_slots, _lists, _ring, _ct); + _allWorkers = Task.WhenAll(_state.Workers); + } + + // The chunk's list goes back to the workers' pool only once the consumer has moved past its last row. + private bool FinishChunk() + { + CsvChunkResult chunk = _chunk!; + _chunk = null; + _list = null; + List models = chunk.Models; + models.Clear(); + _lists.Add(models); + _confirmedNextStart = _parent.AdvanceOrThrow(chunk, _recordsBefore); + _recordsBefore += chunk.Records; + return _confirmedNextStart < long.MaxValue; + } + + public async ValueTask DisposeAsync() + { + if (_done) + { + return; + } + _done = true; + _list = null; + _chunk = null; + if (_cts is null) + { + return; + } + await _cts.CancelAsync().ConfigureAwait(false); + await _allWorkers.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + _parent._lease?.Release(); + _slots?.Dispose(); + _cts.Dispose(); + } + } } } diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvFactory.cs b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvFactory.cs index 80b1e0cb..f17e0e29 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvFactory.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvFactory.cs @@ -22,7 +22,7 @@ internal static IAsyncEnumerable Create(CsvReader reader, int degreeOfPara int dop = Normalize(degreeOfParallelism); if (!reader.TryGetChunkSource(out CsvChunkSource source) || !CanPartition(dop, source.Length, reader.Options)) { - return new CsvEnumerable(reader.FirstSheet, parser.Config, parser.CsvInfo, ownedReader: null, ct); + return new CsvEnumerable(reader.FirstSheet, parser.Config, parser.CsvInfo, ct); } return Build(source, reader.Options, parser, dop, chunkSizeOverride: 0, ct, reader.Lifetime, reader); } @@ -49,7 +49,7 @@ private static IAsyncEnumerable Build( int chunkSizeOverride, CancellationToken ct, ReaderLifetime? lease = null, - object? owner = null) + CsvReader? owner = null) { ExcelParserOptions config = parser.Config; TypeMapInfo info = parser.CsvInfo; @@ -68,16 +68,17 @@ private static IAsyncEnumerable Build( CsvChunkPlan plan = CsvChunkPlan.Create(dataStart, source.Length - dataStart, dop, chunkSizeOverride); if (plan.Count <= 1) { - return Sequential(source.OpenReader(options), parser, ct); + // Without an owner the source is in memory, so the reader opened here holds nothing to release. + return Sequential(owner ?? source.OpenReader(options), parser, ct); } - return new ParallelCsvEnumerable(source, plan, dataStart, map, info, options, config, dop, lease, owner); + return new ParallelCsvEnumerable(source, plan, dataStart, map, info, options, config, dop, lease, owner, ct); } private static CsvEnumerable Sequential( CsvReader reader, ExcelParser parser, CancellationToken ct) { - return new CsvEnumerable(reader.Sheets[0], parser.Config, parser.CsvInfo, ownedReader: reader, ct); + return new CsvEnumerable(reader.FirstSheet, parser.Config, parser.CsvInfo, ct); } private static async IAsyncEnumerable Empty() diff --git a/src/ExcelReader.Core/Reader/Cell.cs b/src/ExcelReader.Core/Reader/Cell.cs index 4cc8a2e5..ea284314 100644 --- a/src/ExcelReader.Core/Reader/Cell.cs +++ b/src/ExcelReader.Core/Reader/Cell.cs @@ -95,15 +95,21 @@ public bool TryGetDouble(out double value) /// /// /// The format provider used for text parsing and for deciding whether '.' is the decimal separator; - /// null means , the culture XLSX and XLSB store numbers in. + /// null means . Ignored for and + /// cells, whose text is a workbook-stored number and always invariant. /// /// The parsed value, when this method returns true. [SkipLocalsInit] public bool TryParse(IFormatProvider? provider, [MaybeNullWhen(false)] out T result) where T : IUtf8SpanParsable { - provider ??= CultureInfo.InvariantCulture; + provider = TextProvider(provider); if (!_hasNumber) { + if (Value.IsEmpty && NeverParsesEmpty()) + { + result = default; + return false; + } if (typeof(T) == typeof(double) && UsesDotDecimalSeparator(provider) && FastDouble.TryParse(Value, out double fast)) { result = Unsafe.As(ref fast); @@ -170,6 +176,25 @@ public bool TryParse(IFormatProvider? provider, [MaybeNullWhen(false)] out T : T.TryParse(Value, provider, out result); } + // Types whose BCL parser rejects an empty span under any provider; a custom IUtf8SpanParsable type may accept one. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool NeverParsesEmpty() + { + return typeof(T) == typeof(double) || typeof(T) == typeof(float) || typeof(T) == typeof(Half) || typeof(T) == typeof(decimal) + || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(short) || typeof(T) == typeof(sbyte) + || typeof(T) == typeof(uint) || typeof(T) == typeof(ulong) || typeof(T) == typeof(ushort) || typeof(T) == typeof(byte) + || typeof(T) == typeof(Guid); + } + + internal IFormatProvider TextProvider(IFormatProvider? provider) + { + if (Type is CellType.Number or CellType.Date) + { + return CultureInfo.InvariantCulture; + } + return provider ?? CultureInfo.InvariantCulture; + } + private static bool TryParseDecimalText(ReadOnlySpan utf8, IFormatProvider provider, out decimal value) { return (UsesDotDecimalSeparator(provider) && FastDecimal.TryParse(utf8, out value)) @@ -376,6 +401,11 @@ private static bool UsesDotDecimalSeparator(IFormatProvider provider) /// The parsed date, when this method returns true. public bool TryGetDateTime(bool isDate1904, out DateTime result) { + if (!_hasNumber && Value.IsEmpty) + { + result = default; + return false; + } if (!_hasNumber && FastDate.TryParse(Value, out result)) { return true; diff --git a/src/ExcelReader.Core/Reader/Columnar/TypedColumnBuilder.cs b/src/ExcelReader.Core/Reader/Columnar/TypedColumnBuilder.cs index 08c9005f..1cb8ec72 100644 --- a/src/ExcelReader.Core/Reader/Columnar/TypedColumnBuilder.cs +++ b/src/ExcelReader.Core/Reader/Columnar/TypedColumnBuilder.cs @@ -71,6 +71,23 @@ internal bool AppendFrom(in Cell cell, bool isDate1904) }; } + // A blank cell in a nullable column: the null the failed parse would have recorded, without parsing. + internal bool TryAppendNull() + { + if (!_nullable) + { + return false; + } + return _type switch + { + ExcelColumnType.StringColumn => AppendNull(_stringOffsets, _stringData.Count), + ExcelColumnType.StringDictionaryColumn or ExcelColumnType.DateColumn => AppendNull(_ints, 0), + ExcelColumnType.Float64Column => AppendNull(_doubles, 0d), + ExcelColumnType.BoolColumn => AppendNull(_bools, (byte)0), + _ => AppendNull(_longs, 0L), + }; + } + // The common cases read straight from the cell's descriptor, with the same result AppendFrom // gives them: text bytes as they are, a binary number as Cell.TryParse returns it. // Everything else returns false and goes through AppendFrom. diff --git a/src/ExcelReader.Core/Reader/Columnar/TypedTableBuilder.cs b/src/ExcelReader.Core/Reader/Columnar/TypedTableBuilder.cs index e9ed4f8e..6b32b778 100644 --- a/src/ExcelReader.Core/Reader/Columnar/TypedTableBuilder.cs +++ b/src/ExcelReader.Core/Reader/Columnar/TypedTableBuilder.cs @@ -25,6 +25,10 @@ internal bool TryAppend(in Row row, int[] columnIndices, bool isDate1904, out in for (int i = 0; i < _columns.Length; i++) { int index = row.IndexOf(columnIndices[i]); + if ((index < 0 || cells[index] is { Type: CellType.Empty, Length: 0, HasNumber: false }) && _columns[i].TryAppendNull()) + { + continue; + } if (index >= 0 && _columns[i].CanAppendDirect(in cells[index]) && _columns[i].TryAppendDirect(in cells[index], in row)) { continue; diff --git a/src/ExcelReader.Core/Reader/Excel.cs b/src/ExcelReader.Core/Reader/Excel.cs index 3bd4b1ce..2fcb05bd 100644 --- a/src/ExcelReader.Core/Reader/Excel.cs +++ b/src/ExcelReader.Core/Reader/Excel.cs @@ -636,7 +636,7 @@ private static void RequireOpenableFormat(ExcelFileFormat format) /// Detects a workbook's file format (XLSX/XLSB/XLS/unknown) from a seekable stream's signature, without consuming it. /// A seekable stream containing the workbook data. The stream's position is restored after detection. Reading starts at the stream's current , which is treated as the start of the workbook; bytes before it are ignored. - /// The detected , or if the signature matches no supported format or the ZIP is corrupt or holds no XLSX/XLSB workbook part. + /// The detected , or if the signature matches no supported format or the ZIP is corrupt, cannot be read (encrypted entries, an unsupported compression method, more entries than the default limit) or holds no XLSX/XLSB workbook part. /// is . /// does not support seeking. public static ExcelFileFormat DetectFileFormat(Stream stream) @@ -651,7 +651,7 @@ public static ExcelFileFormat DetectFileFormat(Stream stream) zip?.Dispose(); return format; } - catch (InvalidDataException) + catch (Exception ex) when (IsNotAReadableWorkbook(ex)) { return ExcelFileFormat.Unknown; } @@ -668,7 +668,7 @@ public static ExcelFileFormat DetectFileFormat(Stream stream) /// Detects a workbook's file format (XLSX/XLSB/XLS/unknown) from an in-memory buffer's signature, without consuming it. /// /// A buffer containing the workbook data. - /// The detected , or if the signature matches no supported format or the ZIP is corrupt or holds no XLSX/XLSB workbook part. + /// The detected , or if the signature matches no supported format or the ZIP is corrupt, cannot be read (encrypted entries, an unsupported compression method, more entries than the default limit) or holds no XLSX/XLSB workbook part. public static ExcelFileFormat DetectFileFormat(ReadOnlyMemory data) { try @@ -677,7 +677,7 @@ public static ExcelFileFormat DetectFileFormat(ReadOnlyMemory data) memZip?.Dispose(); return format; } - catch (InvalidDataException) + catch (Exception ex) when (IsNotAReadableWorkbook(ex)) { return ExcelFileFormat.Unknown; } @@ -686,7 +686,7 @@ public static ExcelFileFormat DetectFileFormat(ReadOnlyMemory data) /// Asynchronously detects a workbook's file format (XLSX/XLSB/XLS/unknown) from a seekable stream's signature, without consuming it. /// A seekable stream containing the workbook data. The stream's position is restored after detection. Reading starts at the stream's current , which is treated as the start of the workbook; bytes before it are ignored. /// A token to cancel the detection operation. - /// The detected , or if the signature matches no supported format or the ZIP is corrupt or holds no XLSX/XLSB workbook part. + /// The detected , or if the signature matches no supported format or the ZIP is corrupt, cannot be read (encrypted entries, an unsupported compression method, more entries than the default limit) or holds no XLSX/XLSB workbook part. /// is . /// does not support seeking. public static async ValueTask DetectFileFormatAsync(Stream stream, CancellationToken ct = default) @@ -701,7 +701,7 @@ public static async ValueTask DetectFileFormatAsync(Stream stre zip?.Dispose(); return format; } - catch (InvalidDataException) + catch (Exception ex) when (IsNotAReadableWorkbook(ex)) { return ExcelFileFormat.Unknown; } @@ -715,6 +715,11 @@ public static async ValueTask DetectFileFormatAsync(Stream stre } } + private static bool IsNotAReadableWorkbook(Exception ex) + { + return ex is InvalidDataException or NotSupportedException or ExcelLimitExceededException; + } + private static IExcelWorkbook OpenSeekable(Stream stream, bool leaveOpen, ExcelReaderOptions? options) { stream = RebaseToPosition(stream, ref leaveOpen); diff --git a/src/ExcelReader.Core/Reader/Internal/StreamSpooler.cs b/src/ExcelReader.Core/Reader/Internal/StreamSpooler.cs new file mode 100644 index 00000000..11d750bb --- /dev/null +++ b/src/ExcelReader.Core/Reader/Internal/StreamSpooler.cs @@ -0,0 +1,104 @@ +using System.Buffers; + +namespace ExcelReader.Core.Reader.Internal +{ + internal static class StreamSpooler + { + internal const int FirstChunk = 64 * 1024; + internal const int MaxChunk = 1024 * 1024; + + internal static byte[] Spool(Stream stream, int firstChunk = FirstChunk, int maxChunk = MaxChunk) + { + List<(byte[] Buffer, int Length)> chunks = []; + try + { + long total = 0; + int size = firstChunk; + while (true) + { + byte[] buffer = ArrayPool.Shared.Rent(size); + int filled = 0; + int read = 1; + while (read > 0 && filled < size) + { + read = stream.Read(buffer, filled, size - filled); + filled += read; + } + chunks.Add((buffer, filled)); + total += filled; + ThrowIfTooLong(total); + if (read == 0) + { + return Assemble(chunks, (int)total); + } + size = Math.Min(size * 2, maxChunk); + } + } + finally + { + Release(chunks); + } + } + + internal static async ValueTask SpoolAsync(Stream stream, CancellationToken ct, int firstChunk = FirstChunk, int maxChunk = MaxChunk) + { + List<(byte[] Buffer, int Length)> chunks = []; + try + { + long total = 0; + int size = firstChunk; + while (true) + { + byte[] buffer = ArrayPool.Shared.Rent(size); + int filled = 0; + int read = 1; + while (read > 0 && filled < size) + { + read = await stream.ReadAsync(buffer.AsMemory(filled, size - filled), ct).ConfigureAwait(false); + filled += read; + } + chunks.Add((buffer, filled)); + total += filled; + ThrowIfTooLong(total); + if (read == 0) + { + return Assemble(chunks, (int)total); + } + size = Math.Min(size * 2, maxChunk); + } + } + finally + { + Release(chunks); + } + } + + private static void ThrowIfTooLong(long total) + { + if (total > Array.MaxLength) + { + throw new IOException("Stream was too long."); + } + } + + private static byte[] Assemble(List<(byte[] Buffer, int Length)> chunks, int total) + { + byte[] result = GC.AllocateUninitializedArray(total); + int offset = 0; + foreach ((byte[] buffer, int length) in chunks) + { + buffer.AsSpan(0, length).CopyTo(result.AsSpan(offset)); + offset += length; + } + return result; + } + + private static void Release(List<(byte[] Buffer, int Length)> chunks) + { + foreach ((byte[] buffer, int _) in chunks) + { + ArrayPool.Shared.Return(buffer); + } + } + } +} diff --git a/src/ExcelReader.Core/Reader/SheetDataReader.cs b/src/ExcelReader.Core/Reader/SheetDataReader.cs index e0ae6672..15ca344f 100644 --- a/src/ExcelReader.Core/Reader/SheetDataReader.cs +++ b/src/ExcelReader.Core/Reader/SheetDataReader.cs @@ -209,13 +209,13 @@ public Type GetFieldType(int i) } [return: DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicFields | DynamicallyAccessedMemberTypes.PublicProperties)] - private static Type TypeOf(Cell cell) + private Type TypeOf(Cell cell) { return cell.Type switch { - CellType.Number => typeof(double), - CellType.Date => typeof(DateTime), - CellType.Boolean => typeof(bool), + CellType.Number when cell.TryGetDouble(out _) => typeof(double), + CellType.Date when cell.TryGetDateTime(_isDate1904, out _) => typeof(DateTime), + CellType.Boolean when ExcelCellReaders.Bool(in cell, default, out _) => typeof(bool), _ => typeof(string), }; } diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSource.cs b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs index d123dedd..9638c0f2 100644 --- a/src/ExcelReader.Core/Reader/Sources/ByteSource.cs +++ b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs @@ -1,3 +1,4 @@ +using ExcelReader.Core.Reader.Internal; using Microsoft.Win32.SafeHandles; namespace ExcelReader.Core.Reader.Sources @@ -88,9 +89,7 @@ private static ByteSource Open(Stream stream, bool leaveOpen, bool fromPosition) } try { - using MemoryStream copy = new(); - stream.CopyTo(copy); - return new MemoryByteSource(copy.GetBuffer().AsMemory(0, (int)copy.Length), owner: null); + return new MemoryByteSource(StreamSpooler.Spool(stream), owner: null); } finally { @@ -114,9 +113,7 @@ private static async ValueTask OpenAsync(Stream stream, bool leaveOp } try { - using MemoryStream copy = new(); - await stream.CopyToAsync(copy, ct).ConfigureAwait(false); - return new MemoryByteSource(copy.GetBuffer().AsMemory(0, (int)copy.Length), owner: null); + return new MemoryByteSource(await StreamSpooler.SpoolAsync(stream, ct).ConfigureAwait(false), owner: null); } finally { diff --git a/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs b/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs index e7c51ffd..53fac66c 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs @@ -61,13 +61,11 @@ private static (Stream Source, bool OwnsSource) EnsureSeekable(Stream stream, bo { return (stream, !leaveOpen); } - MemoryStream ms = new(); - stream.CopyTo(ms); + MemoryStream ms = new(StreamSpooler.Spool(stream), writable: false); if (!leaveOpen) { stream.Dispose(); } - ms.Position = 0; return (ms, true); } @@ -77,13 +75,11 @@ private static (Stream Source, bool OwnsSource) EnsureSeekable(Stream stream, bo { return (stream, !leaveOpen); } - MemoryStream ms = new(); - await stream.CopyToAsync(ms, ct).ConfigureAwait(false); + MemoryStream ms = new(await StreamSpooler.SpoolAsync(stream, ct).ConfigureAwait(false), writable: false); if (!leaveOpen) { await stream.DisposeAsync().ConfigureAwait(false); } - ms.Position = 0; return (ms, true); } diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.cs index 9473fbb1..662357a1 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbWorkbook.cs @@ -266,12 +266,17 @@ public bool TryGetTable(ReadOnlySpan name, [MaybeNullWhen(false)] out Exce } internal Enumerator OpenSheet(int index) + { + return OpenSheet(index, CancellationToken.None); + } + + private Enumerator OpenSheet(int index, CancellationToken ct) { Lifetime.Acquire(this); try { ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![index].Path); - return new Enumerator(this, _zip!.OpenEntryStream(entry, _decompressedBytes.ForRead(), _options), entry.UncompressedSize); + return new Enumerator(this, _zip!.OpenEntryStream(entry, _decompressedBytes.ForRead(), _options), entry.UncompressedSize, ct); } catch { @@ -315,7 +320,7 @@ internal Enumerator OpenSheetAsync(int index, CancellationToken ct) { if (_zip!.HasMemory) { - return OpenSheet(index); + return OpenSheet(index, ct); } Lifetime.Acquire(this); try diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs index 45180fa9..8390de75 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs @@ -1137,7 +1137,7 @@ private void EmitScalarValueFast(Kind kind, ReadOnlySpan v, int valueStart private void EmitIsoDate(ReadOnlySpan v, int col, int style) { - if (TryParseIsoDate(v, out DateTime dt)) + if (TryParseIsoDate(v, _reader.IsDate1904, out DateTime dt)) { if (!ExcelEpoch.TryOADateToSerial(dt.ToOADate(), _reader.IsDate1904, out double serial)) { @@ -1160,7 +1160,7 @@ private void EmitIsoDate(ReadOnlySpan v, int col, int style) } [SkipLocalsInit] - private static bool TryParseIsoDate(ReadOnlySpan utf8, out DateTime value) + private static bool TryParseIsoDate(ReadOnlySpan utf8, bool date1904, out DateTime value) { if (utf8.Length is 0 or > 40) { @@ -1184,8 +1184,15 @@ private static bool TryParseIsoDate(ReadOnlySpan utf8, out DateTime value) value = default; return false; } - value = utf8.Contains((byte)':') && utf8.IndexOfAny((byte)'-', (byte)'/') < 0 ? new DateTime(1899, 12, 31) + parsed.TimeOfDay : parsed.DateTime; - return value.Year >= 100 || RejectDate(out value); + int colon = utf8.IndexOf((byte)':'); + bool timeOnly = colon >= 0 && utf8[..colon].IndexOfAny((byte)'-', (byte)'/') < 0; + if (!timeOnly) + { + value = parsed.DateTime; + return value.Year >= 100 || RejectDate(out value); + } + value = (date1904 ? new DateTime(1904, 1, 1) : new DateTime(1899, 12, 31)) + parsed.TimeOfDay; + return true; } private static bool RejectDate(out DateTime value) diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.cs index 0469029f..70db325a 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.cs @@ -203,13 +203,18 @@ public bool TryGetTable(ReadOnlySpan name, [MaybeNullWhen(false)] out Exce } internal Enumerator OpenSheet(int index) + { + return OpenSheet(index, CancellationToken.None); + } + + private Enumerator OpenSheet(int index, CancellationToken ct) { Lifetime.Acquire(this); try { EnsureSharedLoaded(); ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip, _sheets[index].Path); - return new Enumerator(this, _zip.OpenEntryStream(entry, _decompressedBytes.ForRead(), _options), entry.UncompressedSize); + return new Enumerator(this, _zip.OpenEntryStream(entry, _decompressedBytes.ForRead(), _options), entry.UncompressedSize, ct); } catch { @@ -254,7 +259,7 @@ internal Enumerator OpenSheetAsync(int index, CancellationToken ct) { if (_zip.HasMemory) { - return OpenSheet(index); + return OpenSheet(index, ct); } Lifetime.Acquire(this); try diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxXml.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxXml.cs index 66ee7d99..9453a305 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxXml.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxXml.cs @@ -425,10 +425,19 @@ internal static ReadOnlySpan DetectElementPrefix(ReadOnlySpan src) { return default; } - if (src[i] is (byte)'?' or (byte)'!' or (byte)'/') + if (src[i] == (byte)'/') { continue; } + if (src[i] is (byte)'?' or (byte)'!') + { + i = MarkupEnd(src, i); + if (i < 0) + { + return default; + } + continue; + } int nameStart = i; for (int j = i; j < src.Length; j++) { @@ -446,6 +455,33 @@ internal static ReadOnlySpan DetectElementPrefix(ReadOnlySpan src) } } + private static int MarkupEnd(ReadOnlySpan src, int i) + { + ReadOnlySpan rest = src[i..]; + if (src[i] == (byte)'?') + { + int pi = rest.IndexOf("?>"u8); + return pi < 0 ? -1 : i + pi + 2; + } + if (rest.StartsWith("!--"u8)) + { + int comment = rest[3..].IndexOf("-->"u8); + return comment < 0 ? -1 : i + comment + 6; + } + int depth = 0; + for (int j = 0; j < rest.Length; j++) + { + byte c = rest[j]; + depth += c == (byte)'[' ? 1 : 0; + depth -= c == (byte)']' ? 1 : 0; + if (c == (byte)'>' && depth <= 0) + { + return i + j + 1; + } + } + return -1; + } + internal static byte[] Token(ReadOnlySpan lead, ReadOnlySpan prefix, ReadOnlySpan rest) { byte[] token = new byte[lead.Length + prefix.Length + rest.Length]; diff --git a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs index 9b6322b1..7be43f8b 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs @@ -378,7 +378,8 @@ internal bool HasEntryWithPrefix(ReadOnlySpan utf8Prefix) ReadOnlySpan directory = _directory.Span; foreach (ref readonly ZipEntryRef candidate in _entries.AsSpan(0, Count)) { - if (directory.Slice(candidate.NameStart, candidate.NameLength).StartsWith(utf8Prefix)) + if (candidate.NameLength >= utf8Prefix.Length + && EqualsIgnoringAsciiCase(directory.Slice(candidate.NameStart, utf8Prefix.Length), utf8Prefix)) { return true; } diff --git a/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs b/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs index 603ef9be..51d4cd15 100644 --- a/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs +++ b/src/ExcelReader.Core/Writer/Internal/RecordColumns.cs @@ -4,6 +4,7 @@ using System.Reflection; using System.Runtime.CompilerServices; using ExcelReader.Core.Parser; +using ExcelReader.Core.Parser.Internal; namespace ExcelReader.Core.Writer.Internal { @@ -21,9 +22,7 @@ internal static void WriteRow(TRow row, T record) where TRow : IRowWriter private static PropertyInfo[] FilterProperties() { - return [.. typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) - .Where(prop => prop.GetGetMethod() is not null && prop.GetIndexParameters().Length == 0 - && !Attribute.IsDefined(prop, typeof(ExcelIgnoreAttribute)))]; + return [.. MappableProperties.Of(typeof(T)).Where(static prop => prop.GetGetMethod() is not null)]; } private static string[] BuildHeaders(PropertyInfo[] props) diff --git a/src/ExcelReader.Core/Writer/Xls/OleCompoundWriter.cs b/src/ExcelReader.Core/Writer/Xls/OleCompoundWriter.cs index 74ae486b..8dd25e1b 100644 --- a/src/ExcelReader.Core/Writer/Xls/OleCompoundWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/OleCompoundWriter.cs @@ -390,8 +390,8 @@ private static byte[] BuildDifat(int fatCount, int difatCount, int firstDifatSec internal static ValueTask WriteAsync(Stream destination, int workbookSize, Func writeBody, CancellationToken ct) { - int storedSize = Math.Max(CeilingDiv(workbookSize, SectorSize) * SectorSize, MiniCutoff); - int padding = storedSize - workbookSize; + long storedSize = Math.Max((long)CeilingDivLong(workbookSize, SectorSize) * SectorSize, MiniCutoff); + int padding = (int)(storedSize - workbookSize); CfbStreamSpec spec = new( "Workbook", storedSize, @@ -406,8 +406,8 @@ internal static ValueTask WriteAsync(Stream destination, int workbookSize, Func< internal static void Write(Stream destination, int workbookSize, Action writeBody) { - int storedSize = Math.Max(CeilingDiv(workbookSize, SectorSize) * SectorSize, MiniCutoff); - int padding = storedSize - workbookSize; + long storedSize = Math.Max((long)CeilingDivLong(workbookSize, SectorSize) * SectorSize, MiniCutoff); + int padding = (int)(storedSize - workbookSize); CfbStreamSpec spec = new( "Workbook", storedSize, diff --git a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs index 5aa0583f..a14f3176 100644 --- a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs @@ -282,6 +282,7 @@ internal void NotifyRowEnded() internal void EmitNumber(int row, int col, double value) { ValidateColumn(col); + CellValueGuards.ThrowIfNonFinite(value, nameof(value)); int abstractStyle = EffectiveStyle(col); WriteNumberCell(row, col, abstractStyle == 0 ? XlsGlobals.GeneralXf : XlsGlobals.CustomXf(abstractStyle), value); Track(row, col); @@ -290,8 +291,8 @@ internal void EmitNumber(int row, int col, double value) internal void EmitDate(int row, int col, DateTime value) { ValidateColumn(col); - int abstractStyle = EffectiveStyle(col); double serial = ExcelEpoch.OADateToSerial(value.ToOADate(), _date1904); + int abstractStyle = EffectiveStyle(col); WriteNumberCell(row, col, abstractStyle == 0 ? XlsGlobals.DateXf : XlsGlobals.CustomXf(_owner.DateStyle(abstractStyle)), serial); Track(row, col); } @@ -334,7 +335,6 @@ internal void EmitBlank(int row, int col) private void WriteNumberCell(int row, int col, int xf, double value) { - CellValueGuards.ThrowIfNonFinite(value, nameof(value)); if (!XlsRk.TryEncode(value, out int rk)) { FlushRkRun(); @@ -390,15 +390,23 @@ private static void ValidateColumn(int col) } } + private string TruncatedBaseName(int maxLength) + { + if (_baseName.Length <= maxLength) + { + return _baseName; + } + int length = char.IsHighSurrogate(_baseName[maxLength - 1]) ? maxLength - 1 : maxLength; + return _baseName[..length]; + } + private XlsSheetWriter CreateContinuation() { string contName; for (int n = _owner.SheetCount + 1; ; n++) { string suffix = $" ({n})"; - contName = _baseName.Length + suffix.Length <= MaxSheetNameLength - ? _baseName + suffix - : _baseName[..(MaxSheetNameLength - suffix.Length)] + suffix; + contName = TruncatedBaseName(MaxSheetNameLength - suffix.Length) + suffix; if (_owner.TryClaimSheetName(contName)) { break; diff --git a/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs b/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs index afbb1f98..af6c9fbe 100644 --- a/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs @@ -1,5 +1,6 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; +using ExcelReader.Core.Reader; using ExcelReader.Core.Writer.Internal; using ExcelReader.Core.Writer.Xlsx; @@ -141,7 +142,7 @@ private BiffBuffer BuildGlobals(List layouts, out int workbookSi { globals.PatchI32(offsetPositions[i], offset); layouts[i].Place(offset); - offset += layouts[i].Length; + offset = AdvanceSheetOffset(offset, layouts[i].Length); } workbookSize = offset; return globals; @@ -153,6 +154,17 @@ private BiffBuffer BuildGlobals(List layouts, out int workbookSi } } + // BIFF8 stores sheet and DBCELL positions as 32-bit offsets, and a CFB v3 stream stays under 2 GiB. + internal static int AdvanceSheetOffset(int offset, int sheetLength) + { + long next = (long)offset + sheetLength; + if (next > int.MaxValue) + { + throw new ExcelLimitExceededException("WorkbookStreamBytes", int.MaxValue, next); + } + return (int)next; + } + private void ReleaseStream() { if (_leaveOpen) diff --git a/src/ExcelReader.Core/Writer/Xlsb/XlsbSheetWriter.cs b/src/ExcelReader.Core/Writer/Xlsb/XlsbSheetWriter.cs index d296933b..bc0cf655 100644 --- a/src/ExcelReader.Core/Writer/Xlsb/XlsbSheetWriter.cs +++ b/src/ExcelReader.Core/Writer/Xlsb/XlsbSheetWriter.cs @@ -304,11 +304,17 @@ private XlsbRowWriter StartRowCore(int styleId, int heightTwips) public void WriteRow(ReadOnlySpan values) { BeginRow(styleId: 0, heightTwips: 0); - for (int i = 0; i < values.Length; i++) + try { - WriteCell(i, values[i]); + for (int i = 0; i < values.Length; i++) + { + WriteCell(i, values[i]); + } + } + finally + { + FinishRow(); } - FinishRow(); } internal void NotifyRowEnded() @@ -845,21 +851,22 @@ internal void WriteBoolCell(int columnIndex, bool value) MaybeFlush(); } - internal void WriteDateSerialCell(int columnIndex, double serial) + internal void WriteDateSerialCell(int columnIndex, double oadate) { + double serial = ExcelEpoch.OADateToSerial(oadate, _date1904); int styleId = EffectiveStyle(columnIndex); - WriteDoubleCellCore(columnIndex, ExcelEpoch.OADateToSerial(serial, _date1904), styleId == 0 ? 1 : _owner.DateStyle(styleId)); + WriteDoubleCellCore(columnIndex, serial, styleId == 0 ? 1 : _owner.DateStyle(styleId)); } internal void WriteDoubleCell(int columnIndex, double value) { + CellValueGuards.ThrowIfNonFinite(value, nameof(value)); WriteDoubleCellCore(columnIndex, value, EffectiveStyle(columnIndex)); } private void WriteDoubleCellCore(int columnIndex, double value, int style) { ValidateColumn(columnIndex); - CellValueGuards.ThrowIfNonFinite(value, nameof(value)); TrackColumn(columnIndex); if (XlsRk.TryEncode(value, out int rk)) { diff --git a/src/ExcelReader.Core/Writer/Xlsx/XlsxRowWriter.cs b/src/ExcelReader.Core/Writer/Xlsx/XlsxRowWriter.cs index 3b1d9803..4673a371 100644 --- a/src/ExcelReader.Core/Writer/Xlsx/XlsxRowWriter.cs +++ b/src/ExcelReader.Core/Writer/Xlsx/XlsxRowWriter.cs @@ -38,9 +38,7 @@ private int EffectiveStyle() { if (_pendingStyle >= 0) { - int pending = _pendingStyle; - _pendingStyle = -1; - return pending; + return _pendingStyle; } return _rowStyleId != 0 ? _rowStyleId : _owner.GetColumnStyle(_columnIndex); } @@ -90,8 +88,8 @@ public void WriteUtf8(ReadOnlySpan utf8) ThrowIfDisposed(); if (!_owner.UseSharedStrings && CellFormatter.IsPlainUtf8Text(utf8)) { - CellFormatter.WritePlainUtf8String(_row, utf8, _columnIndex, _rowNumber, ConsumeCellReference(), EffectiveStyle()); - _columnIndex++; + CellFormatter.WritePlainUtf8String(_row, utf8, _columnIndex, _rowNumber, IncludeCellReference(), EffectiveStyle()); + CompleteCell(); return; } Utf8Chars.Decode(utf8, this, static (chars, self) => self.WriteText(chars, owned: null)); @@ -103,12 +101,12 @@ private void WriteText(ReadOnlySpan value, string? owned) if (_owner.UseSharedStrings) { int index = owned is null ? _owner.GetSharedStringIndex(value) : _owner.GetSharedStringIndex(owned); - CellFormatter.WriteSharedString(_row, index, _columnIndex, _rowNumber, ConsumeCellReference(), EffectiveStyle()); - _columnIndex++; + CellFormatter.WriteSharedString(_row, index, _columnIndex, _rowNumber, IncludeCellReference(), EffectiveStyle()); + CompleteCell(); return; } - CellFormatter.WriteString(_row, value, _columnIndex, _rowNumber, ConsumeCellReference(), EffectiveStyle()); - _columnIndex++; + CellFormatter.WriteString(_row, value, _columnIndex, _rowNumber, IncludeCellReference(), EffectiveStyle()); + CompleteCell(); } /// @@ -116,8 +114,8 @@ private void WriteText(ReadOnlySpan value, string? owned) public void Write(bool value) { ThrowIfDisposed(); - CellFormatter.WriteBool(_row, value, _columnIndex, _rowNumber, ConsumeCellReference(), EffectiveStyle()); - _columnIndex++; + CellFormatter.WriteBool(_row, value, _columnIndex, _rowNumber, IncludeCellReference(), EffectiveStyle()); + CompleteCell(); } /// @@ -139,8 +137,8 @@ public void Write(DateTime value) { ThrowIfDisposed(); int styleId = EffectiveStyle(); - CellFormatter.WriteDateTime(_row, value, _owner.Date1904, _columnIndex, _rowNumber, ConsumeCellReference(), styleId == 0 ? 1 : _owner.DateStyle(styleId)); - _columnIndex++; + CellFormatter.WriteDateTime(_row, value, _owner.Date1904, _columnIndex, _rowNumber, IncludeCellReference(), styleId == 0 ? 1 : _owner.DateStyle(styleId)); + CompleteCell(); } /// @@ -165,8 +163,8 @@ public void Write(DateOnly value) { ThrowIfDisposed(); int styleId = EffectiveStyle(); - CellFormatter.WriteDateTime(_row, value.ToDateTime(TimeOnly.MinValue), _owner.Date1904, _columnIndex, _rowNumber, ConsumeCellReference(), styleId == 0 ? 1 : _owner.DateStyle(styleId)); - _columnIndex++; + CellFormatter.WriteDateTime(_row, value.ToDateTime(TimeOnly.MinValue), _owner.Date1904, _columnIndex, _rowNumber, IncludeCellReference(), styleId == 0 ? 1 : _owner.DateStyle(styleId)); + CompleteCell(); } /// @@ -191,8 +189,8 @@ public void Write(DateOnly? value) public void Write(TimeOnly value) { ThrowIfDisposed(); - CellFormatter.WriteNumber(_row, value.Ticks / (double)TimeSpan.TicksPerDay, _columnIndex, _rowNumber, ConsumeCellReference(), EffectiveStyle()); - _columnIndex++; + CellFormatter.WriteNumber(_row, value.Ticks / (double)TimeSpan.TicksPerDay, _columnIndex, _rowNumber, IncludeCellReference(), EffectiveStyle()); + CompleteCell(); } /// @@ -214,8 +212,8 @@ public void Write(T value) where T : IUtf8SpanFormattable { ThrowIfDisposed(); - CellFormatter.WriteNumber(_row, value, _columnIndex, _rowNumber, ConsumeCellReference(), EffectiveStyle()); - _columnIndex++; + CellFormatter.WriteNumber(_row, value, _columnIndex, _rowNumber, IncludeCellReference(), EffectiveStyle()); + CompleteCell(); } /// @@ -255,18 +253,23 @@ public void Skip(int count = 1) _columnIndex += count; } - private bool ConsumeCellReference() + private bool IncludeCellReference() { ExcelLimits.ThrowIfColumnOutOfRange(_columnIndex); - bool result = _useCellReferences; + return _useCellReferences; + } + + private void CompleteCell() + { + _pendingStyle = -1; _useCellReferences = false; - return result; + _columnIndex++; } private void WriteEmptyCell() { - CellFormatter.WriteEmpty(_row, _columnIndex, _rowNumber, includeReference: ConsumeCellReference(), styleId: EffectiveStyle()); - _columnIndex++; + CellFormatter.WriteEmpty(_row, _columnIndex, _rowNumber, includeReference: IncludeCellReference(), styleId: EffectiveStyle()); + CompleteCell(); } /// diff --git a/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs b/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs index 7a977bde..197b6fd2 100644 --- a/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs +++ b/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs @@ -26,6 +26,15 @@ internal sealed class DeflateEncoder : IDisposable // each symbol writes ahead of the position it advances to. private const int MaxEncodedBlock = (MaxBlockLength * 9 / 8) + 24; + // A table reused from below this base leaves a stream 3 GiB before its positions wrap. + private const uint MaxTableBase = 1u << 30; + + [ThreadStatic] + private static HashTable? _threadTable; + + internal static bool AlwaysClearTable; + + private readonly HashTable _hash; private byte[] _window = []; private uint[] _table; private int _encoded; @@ -34,10 +43,24 @@ internal sealed class DeflateEncoder : IDisposable private ulong _bits; private int _bitCount; + // Slots from earlier streams sit below _windowStart, so the distance check rejects them as it rejects zeros. internal DeflateEncoder() { - _table = ArrayPool.Shared.Rent(1 << HashBits); - Array.Clear(_table, 0, 1 << HashBits); + _hash = _threadTable ?? new HashTable(); + _threadTable = null; + if (_hash.Base >= MaxTableBase || AlwaysClearTable) + { + Array.Clear(_hash.Slots); + _hash.Base = 0; + } + _table = _hash.Slots; + _windowStart = _hash.Base; + } + + internal static void SetThreadTableBase(uint value) + { + _threadTable ??= new HashTable(); + _threadTable.Base = value; } /// @@ -107,11 +130,23 @@ public void Dispose() { ArrayPool.Shared.Return(_window); } - ArrayPool.Shared.Return(_table); + ReleaseTable(); _window = []; _table = []; } + // A slid stream may have wrapped its positions; an interrupted block may have hashed past _encoded. + private void ReleaseTable() + { + _threadTable ??= _hash; + if (_windowStart != _hash.Base) + { + _hash.Base = MaxTableBase; + return; + } + _hash.Base = _windowStart + (uint)_encoded + MaxBlockLength; + } + private void FlushSync(ReadOnlySpan source, BiffBuffer output) { EncodeBlock(source, source.Length, final: false, output); @@ -286,6 +321,12 @@ private static int MatchLength(ref byte window, int candidate, int current, int return length; } + private sealed class HashTable + { + internal readonly uint[] Slots = new uint[1 << HashBits]; + internal uint Base; + } + [StructLayout(LayoutKind.Auto)] private struct BitState { diff --git a/src/ExcelReader.Generator/ExcelRowMapGenerator.cs b/src/ExcelReader.Generator/ExcelRowMapGenerator.cs index 80d66619..6cefc021 100644 --- a/src/ExcelReader.Generator/ExcelRowMapGenerator.cs +++ b/src/ExcelReader.Generator/ExcelRowMapGenerator.cs @@ -283,8 +283,12 @@ private static IEnumerable CollectMappableProperties(INamedType { foreach (IPropertySymbol property in current.GetMembers().OfType() .Where(p => !p.IsStatic && p.Parameters.Length == 0 && p.DeclaredAccessibility == Accessibility.Public - && !HasAttribute(p, IgnoreAttribute) && seenNames.Add(p.Name))) + && seenNames.Add(p.Name))) { + if (HasAttribute(property, IgnoreAttribute)) + { + continue; + } yield return property; } } @@ -298,14 +302,25 @@ private static bool IsPartial(INamedTypeSymbol symbol) .Any(static d => d.Modifiers.Any(SyntaxKind.PartialKeyword)); } + private static IEnumerable InheritedAttributes(IPropertySymbol property) + { + for (IPropertySymbol? current = property; current is not null; current = current.OverriddenProperty) + { + foreach (AttributeData attribute in current.GetAttributes()) + { + yield return attribute; + } + } + } + private static bool HasAttribute(IPropertySymbol property, string fullName) { - return property.GetAttributes().Any(a => string.Equals(a.AttributeClass?.ToDisplayString(), fullName, StringComparison.Ordinal)); + return InheritedAttributes(property).Any(a => string.Equals(a.AttributeClass?.ToDisplayString(), fullName, StringComparison.Ordinal)); } private static PropertyPlan BuildPlan(List diagnostics, INamedTypeSymbol owner, IPropertySymbol property, ref bool hadPropertyError) { - string[] names = [.. property.GetAttributes() + string[] names = [.. InheritedAttributes(property) .Where(a => string.Equals(a.AttributeClass?.ToDisplayString(), ColumnAttribute, StringComparison.Ordinal)) .Select(static a => (string?)a.ConstructorArguments.FirstOrDefault().Value) .Where(static n => n is not null)!]; @@ -314,7 +329,7 @@ private static PropertyPlan BuildPlan(List diagnostics, INamedTy names = [property.Name]; } - AttributeData? required = property.GetAttributes() + AttributeData? required = InheritedAttributes(property) .FirstOrDefault(a => string.Equals(a.AttributeClass?.ToDisplayString(), RequiredAttribute, StringComparison.Ordinal)); bool isRequired = required is not null; bool allowEmpty = required?.NamedArguments.FirstOrDefault(static kv => string.Equals(kv.Key, "AllowEmpty", StringComparison.Ordinal)).Value.Value is true; @@ -331,7 +346,7 @@ private static PropertyPlan BuildPlan(List diagnostics, INamedTy hadPropertyError = true; } - AttributeData? converterAttr = property.GetAttributes() + AttributeData? converterAttr = InheritedAttributes(property) .FirstOrDefault(a => string.Equals(a.AttributeClass?.ToDisplayString(), ConverterAttribute, StringComparison.Ordinal)); string? style = StyleExpression(diagnostics, property, ref hadPropertyError); @@ -478,7 +493,7 @@ private static string ColumnEmit(string header, string write, string? style) // The attribute's own ToCellStyle maps it to a CellStyle, so the generated map and reflection agree. private static string? StyleExpression(List diagnostics, IPropertySymbol property, ref bool hadPropertyError) { - AttributeData? attribute = property.GetAttributes() + AttributeData? attribute = InheritedAttributes(property) .FirstOrDefault(a => string.Equals(a.AttributeClass?.ToDisplayString(), StyleAttribute, StringComparison.Ordinal)); if (attribute is null) { diff --git a/src/ExcelReader.Native/Exports.Arrow.cs b/src/ExcelReader.Native/Exports.Arrow.cs index bc69f61d..c5050e2e 100644 --- a/src/ExcelReader.Native/Exports.Arrow.cs +++ b/src/ExcelReader.Native/Exports.Arrow.cs @@ -11,6 +11,14 @@ public static int ParseArrow(nint handle, int sheet, NativeColumnSpecRaw* specs, int degreeOfParallelism, ArrowArray* outArray, ArrowSchema* outSchema) { NativeApi.ClearLastError(); + if (outArray is not null) + { + *outArray = default; + } + if (outSchema is not null) + { + *outSchema = default; + } if (specs is null || outArray is null || outSchema is null || !TypedApi.IsValidSpecCount(specCount)) { diff --git a/src/ExcelReader.Native/Exports.Reading.cs b/src/ExcelReader.Native/Exports.Reading.cs index 095495b0..00ac612d 100644 --- a/src/ExcelReader.Native/Exports.Reading.cs +++ b/src/ExcelReader.Native/Exports.Reading.cs @@ -37,7 +37,7 @@ public static int OpenMemory(byte* data, int dataLength, int format, NativeOpenO *outHandle = 0; } - if (!IsValidOpenRequest(data, dataLength, outHandle)) + if (!IsValidOpenRequest(data, dataLength, outHandle, allowEmpty: true)) { return NativeStatus.InvalidArgument; } @@ -329,6 +329,14 @@ public static int RowsNextView(nint rows, NativeRow* outRow) public static int RowsNextCells(nint rows, NativeCellView** outCells, int* outCount) { NativeApi.ClearLastError(); + if (outCells is not null) + { + *outCells = null; + } + if (outCount is not null) + { + *outCount = 0; + } if (outCells is null || outCount is null) { @@ -433,9 +441,14 @@ public static void FreeSchema(NativeInferredSchema* schema) } } - private static bool IsValidOpenRequest(byte* source, int sourceLength, nint* outHandle) + private static bool IsValidOpenRequest(byte* source, int sourceLength, nint* outHandle, bool allowEmpty = false) { - return source is not null && sourceLength >= 0 && outHandle is not null; + bool valid = (source is not null || (allowEmpty && sourceLength == 0)) && sourceLength >= 0 && outHandle is not null; + if (!valid) + { + NativeApi.SetLastError("the source must not be NULL (unless its length is 0 for an in-memory open), its length must not be negative and out_handle must not be NULL."); + } + return valid; } private static bool TryReadOpenOptions(NativeOpenOptionsRaw* options, out NativeOpenOptionsRaw? rawOptions) diff --git a/src/ExcelReader.Native/Exports.Typed.cs b/src/ExcelReader.Native/Exports.Typed.cs index dfb03947..f69f3b2f 100644 --- a/src/ExcelReader.Native/Exports.Typed.cs +++ b/src/ExcelReader.Native/Exports.Typed.cs @@ -10,6 +10,10 @@ public static int ParseTyped(nint handle, int sheet, NativeColumnSpecRaw* specs, int degreeOfParallelism, NativeTable* outTable) { NativeApi.ClearLastError(); + if (outTable is not null) + { + *outTable = default; + } if (specs is null || outTable is null || !TypedApi.IsValidSpecCount(specCount)) { diff --git a/src/ExcelReader.Native/Exports.Writer.cs b/src/ExcelReader.Native/Exports.Writer.cs index eced877a..5a003927 100644 --- a/src/ExcelReader.Native/Exports.Writer.cs +++ b/src/ExcelReader.Native/Exports.Writer.cs @@ -42,11 +42,14 @@ public static int WriteTyped(byte* path, int pathLength, int format, NativeColum public static int WriteTypedToMemory(int format, NativeColumnSpecRaw* specs, NativeTable* table, NativeWriteOptionsRaw* options, NativeBuffer* outBuffer) { NativeApi.ClearLastError(); + if (outBuffer is not null) + { + *outBuffer = default; + } if (specs is null || table is null || outBuffer is null || !TypedApi.IsValidSpecCount(table->ColumnCount)) { return NativeStatus.InvalidArgument; } - *outBuffer = default; try { @@ -99,11 +102,14 @@ private static bool TryDecodeWriteOptions(NativeWriteOptionsRaw* options, out Na public static int OpenWriteHandle(byte* path, int pathLength, int format, NativeWriteOptionsRaw* options, nint* outHandle) { NativeApi.ClearLastError(); + if (outHandle is not null) + { + *outHandle = 0; + } if (!IsValidOpenRequest(path, pathLength, outHandle)) { return NativeStatus.InvalidArgument; } - *outHandle = 0; if (!TryDecodeWriteOptions(options, out NativeWriteOptions decodedOptions)) { return NativeStatus.InvalidArgument; @@ -351,12 +357,15 @@ public static int AbortWriteHandle(nint handle) public static int AddStyle(nint handle, NativeCellStyleRaw* style, int* outStyleId) { NativeApi.ClearLastError(); + if (outStyleId is not null) + { + *outStyleId = 0; + } if (style is null || outStyleId is null) { NativeApi.SetLastError("xl_add_style needs a style and an out_style_id."); return NativeStatus.InvalidArgument; } - *outStyleId = 0; if (!NativeCellStyle.TryDecode(style, out CellStyle? decoded, out string? error)) { NativeApi.SetLastError(error); diff --git a/src/ExcelReader.Native/Exports.cs b/src/ExcelReader.Native/Exports.cs index efe7053b..02c437f6 100644 --- a/src/ExcelReader.Native/Exports.cs +++ b/src/ExcelReader.Native/Exports.cs @@ -30,11 +30,14 @@ public static int EncryptPackage(byte* packagePath, int packagePathLength, byte* public static int EncryptPackageToMemory(byte* package, int packageLength, byte* password, int passwordLength, NativeBuffer* outBuffer) { NativeApi.ClearLastError(); + if (outBuffer is not null) + { + *outBuffer = default; + } if (package is null || packageLength <= 0 || password is null || passwordLength <= 0 || outBuffer is null) { return NativeStatus.InvalidArgument; } - *outBuffer = default; try { @@ -109,6 +112,12 @@ private static bool TryDecodeColumnSpecs(NativeColumnSpecRaw* specs, int specCou decoded = []; return false; } + if (raw.Names[n] is null) + { + NativeApi.SetLastError("a column spec name must not be NULL."); + decoded = []; + return false; + } names[n] = Encoding.UTF8.GetString(raw.Names[n], raw.NameLens[n]); } decoded[i] = new NativeColumnSpec diff --git a/src/ExcelReader.NetFramework/Interop/AggregationRun.cs b/src/ExcelReader.NetFramework/Interop/AggregationRun.cs index 7c424769..fa75abd2 100644 --- a/src/ExcelReader.NetFramework/Interop/AggregationRun.cs +++ b/src/ExcelReader.NetFramework/Interop/AggregationRun.cs @@ -18,7 +18,7 @@ internal abstract unsafe class AggregationRun private static readonly IntPtr CombinePointer = Marshal.GetFunctionPointerForDelegate(CombineCallback); private static readonly IntPtr FreeStatePointer = Marshal.GetFunctionPointerForDelegate(FreeStateCallback); - private ExceptionDispatchInfo? _error; + private readonly List _errors = new List(); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate int SeedFn(IntPtr* outState, IntPtr userData); @@ -54,11 +54,7 @@ internal static TAccumulator Run(ExcelWorkbook workbook, int sheet IntPtr state = IntPtr.Zero; int status = NativeMethods.xl_aggregate(handle.DangerousGetHandle(), sheet, &aggregation, headerRow, degreeOfParallelism, &state); object? result = Take(state); - if (status == CallbackFailed) - { - run._error?.Throw(); - } - + run.FailureFor(status)?.Throw(); NativeError.ThrowIfFailed(status); if (result is null) { @@ -168,11 +164,24 @@ private static void OnFreeState(IntPtr state, IntPtr userData) } } - // The native side may re-read a failed chunk and still succeed, so the exception surfaces only if the run fails. + // Each failure gets its own code; native returns the code of the failure the run ended on. private int Fail(Exception exception) { - Interlocked.CompareExchange(ref _error, ExceptionDispatchInfo.Capture(exception), null); - return CallbackFailed; + lock (_errors) + { + _errors.Add(ExceptionDispatchInfo.Capture(exception)); + return CallbackFailed - (_errors.Count - 1); + } + } + + private ExceptionDispatchInfo? FailureFor(int status) + { + if (status <= 0 || CallbackFailed - status >= _errors.Count) + { + return null; + } + + return _errors[CallbackFailed - status]; } } diff --git a/src/ExcelReader.NetFramework/Parser/Internal/MappedProperties.cs b/src/ExcelReader.NetFramework/Parser/Internal/MappedProperties.cs new file mode 100644 index 00000000..29bf0fcb --- /dev/null +++ b/src/ExcelReader.NetFramework/Parser/Internal/MappedProperties.cs @@ -0,0 +1,16 @@ +using System.Reflection; + +namespace ExcelReader.NetFramework.Parser.Internal +{ + internal static class MappedProperties + { + internal static IEnumerable Of(Type type) + { + return type.GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.GetIndexParameters().Length == 0) + .GroupBy(p => p.Name, StringComparer.Ordinal) + .Select(g => g.Aggregate((a, b) => b.DeclaringType!.IsSubclassOf(a.DeclaringType!) ? b : a)) + .Where(p => !Attribute.IsDefined(p, typeof(ExcelIgnoreAttribute))); + } + } +} diff --git a/src/ExcelReader.NetFramework/Parser/Internal/TypeMap.cs b/src/ExcelReader.NetFramework/Parser/Internal/TypeMap.cs index c3698039..0ec477f8 100644 --- a/src/ExcelReader.NetFramework/Parser/Internal/TypeMap.cs +++ b/src/ExcelReader.NetFramework/Parser/Internal/TypeMap.cs @@ -16,13 +16,8 @@ private TypeMap() } List> properties = new List>(); - foreach (PropertyInfo property in typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance)) + foreach (PropertyInfo property in MappedProperties.Of(typeof(T))) { - if (Attribute.IsDefined(property, typeof(ExcelIgnoreAttribute))) - { - continue; - } - if (property.GetSetMethod() is null) { RejectMappedGetOnly(property); diff --git a/src/ExcelReader.NetFramework/Reader/ExcelCellView.cs b/src/ExcelReader.NetFramework/Reader/ExcelCellView.cs index 915a056f..32124aec 100644 --- a/src/ExcelReader.NetFramework/Reader/ExcelCellView.cs +++ b/src/ExcelReader.NetFramework/Reader/ExcelCellView.cs @@ -80,7 +80,10 @@ public double? Number return null; } - return _cell->Number.ToString("R", CultureInfo.InvariantCulture); + double number = _cell->Number; + string shortest = number.ToString("R", CultureInfo.InvariantCulture); + // .NET Framework's "R" can lose the last bit on x64. + return double.Parse(shortest, NumberStyles.Float, CultureInfo.InvariantCulture) == number ? shortest : number.ToString("G17", CultureInfo.InvariantCulture); } } } diff --git a/src/ExcelReader.NetFramework/Writer/Internal/WriteMap.cs b/src/ExcelReader.NetFramework/Writer/Internal/WriteMap.cs index f9588ba3..17de8389 100644 --- a/src/ExcelReader.NetFramework/Writer/Internal/WriteMap.cs +++ b/src/ExcelReader.NetFramework/Writer/Internal/WriteMap.cs @@ -1,6 +1,7 @@ using System.Reflection; using ExcelReader.NetFramework.Interop; using ExcelReader.NetFramework.Parser; +using ExcelReader.NetFramework.Parser.Internal; namespace ExcelReader.NetFramework.Writer.Internal { @@ -10,8 +11,8 @@ internal sealed class WriteMap private WriteMap() { - IEnumerable properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) - .Where(p => p.GetGetMethod() is not null && p.GetIndexParameters().Length == 0 && !Attribute.IsDefined(p, typeof(ExcelIgnoreAttribute))) + IEnumerable properties = MappedProperties.Of(typeof(T)) + .Where(p => p.GetGetMethod() is not null) .OrderBy(p => Depth(p.DeclaringType!)) .ThenBy(p => p.MetadataToken); Columns = properties.Select(Create).ToArray(); diff --git a/tests/ExcelReader.NetFramework.Tests/Reader/ForEachAndAggregateTests.cs b/tests/ExcelReader.NetFramework.Tests/Reader/ForEachAndAggregateTests.cs index a5a952ed..018fb568 100644 --- a/tests/ExcelReader.NetFramework.Tests/Reader/ForEachAndAggregateTests.cs +++ b/tests/ExcelReader.NetFramework.Tests/Reader/ForEachAndAggregateTests.cs @@ -1,6 +1,7 @@ using System.Globalization; using System.Text; using ExcelReader.NetFramework.Reader; +using ExcelReader.NetFramework.Writer; namespace ExcelReader.NetFramework.Tests.Reader { @@ -77,6 +78,30 @@ public void CellViewExposesNumberAndText() Assert.Equal(CsvTexts, texts, StringComparer.Ordinal); } + [Theory] + [InlineData(ExcelFormat.Xlsx)] + [InlineData(ExcelFormat.Xlsb)] + [InlineData(ExcelFormat.Xls)] + public void NumberTextRoundTrips(ExcelFormat format) + { + double[] values = { 0.84551240822557006, 0.1 + 0.2, 1.0 / 3, 5e-324, double.MaxValue }; + using ExcelWorkbookWriter writer = ExcelWorkbookWriter.CreateInMemory(format); + writer.StartSheet("N"); + foreach (double value in values) + { + writer.StartRow(); + writer.Write(value); + writer.EndRow(); + } + + writer.EndSheet(); + using ExcelWorkbook workbook = ExcelWorkbook.Open(writer.CloseToArray(), new ExcelOpenOptions { Format = format }); + List texts = new List(); + workbook.Sheet(0).ForEach(row => texts.Add(row[0].GetText())); + + Assert.Equal(values.Select(v => BitConverter.DoubleToInt64Bits(v)), texts.Select(t => BitConverter.DoubleToInt64Bits(double.Parse(t!, CultureInfo.InvariantCulture)))); + } + [Fact] public void RowViewIndexerRejectsOutOfRange() { @@ -127,6 +152,14 @@ public void AggregateRethrowsTheCallbackException() Assert.Equal("boom", error.Message); } + [Fact] + public void AggregateRethrowsTheFailureASequentialReadHits() + { + using ExcelWorkbook workbook = ExcelWorkbook.Open(LargeCsv(), new ExcelOpenOptions { Format = ExcelFormat.Csv }); + InvalidTimeZoneException error = Assert.Throws(() => workbook.Sheet(0).Aggregate(() => new ThrowingFrom(), degreeOfParallelism: 4)); + Assert.Equal(ThrowingFrom.FirstFailingId.ToString(CultureInfo.InvariantCulture), error.Message); + } + [Fact] public void AggregateRejectsANullSeedResult() { @@ -196,6 +229,29 @@ public void Combine(Ordered other) } } + private sealed class ThrowingFrom : IExcelAccumulator + { + internal const int FirstFailingId = 100; + + public void Accumulate(ExcelRowView row) + { + int id = (int)row[0].Number!.Value; + if (id == FirstFailingId - 1) + { + Thread.Sleep(300); + } + + if (id >= FirstFailingId) + { + throw new InvalidTimeZoneException(id.ToString(CultureInfo.InvariantCulture)); + } + } + + public void Combine(ThrowingFrom other) + { + } + } + private sealed class Throwing : IExcelAccumulator { public void Accumulate(ExcelRowView row) diff --git a/tests/ExcelReader.NetFramework.Tests/Reader/TypedReadTests.cs b/tests/ExcelReader.NetFramework.Tests/Reader/TypedReadTests.cs index 5e61c191..3277dece 100644 --- a/tests/ExcelReader.NetFramework.Tests/Reader/TypedReadTests.cs +++ b/tests/ExcelReader.NetFramework.Tests/Reader/TypedReadTests.cs @@ -90,6 +90,23 @@ public void NullableGetsNullForEmptyCells() Assert.Equal(OneThenNull, rows.Select(r => r.Qty).ToArray()); } + [Fact] + public void IndexerIsNotMapped() + { + using ExcelWorkbook workbook = OpenCsv(QtyCsv); + WithIndexer[] rows = workbook.Sheet(0).Read(); + Assert.Equal(OneThenNull, rows.Select(r => r.Qty).ToArray()); + } + + [Fact] + public void NewPropertyHidesTheBaseOne() + { + using ExcelWorkbook workbook = OpenCsv(QtyCsv); + HidingQty[] rows = workbook.Sheet(0).Read(); + Assert.Equal(OneThenZero, rows.Select(r => r.Qty).ToArray()); + Assert.All(rows, r => Assert.Null(((HiddenQty)r).Qty)); + } + [Fact] public void NonNullableGetsDefaultForEmptyCells() { @@ -392,5 +409,26 @@ public sealed class ProductOnly { public string? Product { get; set; } } + + public sealed class WithIndexer + { + public int? Qty { get; set; } + + public object? this[string key] + { + get => null; + set { } + } + } + + public class HiddenQty + { + public string? Qty { get; set; } + } + + public sealed class HidingQty : HiddenQty + { + public new int Qty { get; set; } + } } } diff --git a/tests/ExcelReader.NetFramework.Tests/Writer/WriteSheetTests.cs b/tests/ExcelReader.NetFramework.Tests/Writer/WriteSheetTests.cs index f9c3d0da..1667f01a 100644 --- a/tests/ExcelReader.NetFramework.Tests/Writer/WriteSheetTests.cs +++ b/tests/ExcelReader.NetFramework.Tests/Writer/WriteSheetTests.cs @@ -56,6 +56,16 @@ public sealed class TimeRow public TimeSpan At { get; set; } } + public class HiddenCode + { + public string? Code { get; set; } + } + + public sealed class HidingCode : HiddenCode + { + public new int Code { get; set; } + } + public sealed class WriteSheetTests { private static readonly string[] Header = { "Nome", "Id", "Qty", "Small", "Tiny", "Price", "Ratio", "Amount", "Active", "When", "At", "MaybeQty", "MaybeWhen" }; @@ -142,6 +152,16 @@ public void HeaderUsesFirstColumnName() Assert.Equal(Header, header.Cells.Select(c => c.Text).ToArray()); } + [Fact] + public void NewPropertyHidesTheBaseOne() + { + using ExcelWorkbookWriter writer = ExcelWorkbookWriter.CreateInMemory(ExcelFormat.Csv); + writer.WriteSheet("H", new[] { new HidingCode { Code = 7 } }); + using ExcelWorkbook workbook = Open(writer.CloseToArray(), ExcelFormat.Csv); + + Assert.Equal("Code|7", string.Join("|", workbook.Sheet(0).ReadRows().SelectMany(r => r.Cells).Select(c => c.Text))); + } + [Fact] public void EmptyListWritesHeaderOnly() { diff --git a/tests/ExcelReader.Tests/Arrow/ArrowConversionTests.cs b/tests/ExcelReader.Tests/Arrow/ArrowConversionTests.cs index e233480b..2e1b0b67 100644 --- a/tests/ExcelReader.Tests/Arrow/ArrowConversionTests.cs +++ b/tests/ExcelReader.Tests/Arrow/ArrowConversionTests.cs @@ -66,6 +66,42 @@ public void ToArrowRecordBatch_Should_Append_Null_For_Blank_Cells_On_Nullable_Co Assert.Equal(1, qty.NullCount); } + [Fact] + public void ToArrowRecordBatch_Should_Infer_Nullable_Columns_When_A_Blank_Follows_The_Sample() + { + using MemoryStream ms = new(); + using (Core.Writer.Xlsb.XlsbWorkbookWriter workbook = Core.Writer.Xlsb.XlsbWorkbookWriter.Create(ms, leaveOpen: true)) + { + using Core.Writer.Xlsb.XlsbSheetWriter sheet = workbook.AddSheet("S"); + using (Core.Writer.Xlsb.XlsbRowWriter header = sheet.StartRow()) + { + header.Write("Amount"); + } + for (int i = 1; i <= 150; i++) + { + using Core.Writer.Xlsb.XlsbRowWriter row = sheet.StartRow(); + if (i == 120) + { + row.Skip(); + continue; + } + row.Write(i); + } + sheet.End(); + workbook.End(); + } + ms.Position = 0; + using var reader = Excel.FromXlsb(ms); + + using RecordBatch batch = reader.FirstSheet.ToArrowRecordBatch(); + + var amount = Assert.IsType(batch.Column(0)); + Assert.Equal(150, batch.Length); + Assert.True(amount.IsNull(119)); + Assert.Equal(1, amount.NullCount); + Assert.Equal(121L, amount.GetValue(120)); + } + [Fact] public void ToArrowRecordBatch_Should_Append_Null_For_Blank_Cells_On_Nullable_String_Columns() { diff --git a/tests/ExcelReader.Tests/Cli/CliTests.cs b/tests/ExcelReader.Tests/Cli/CliTests.cs index ed71d501..36aa0ba4 100644 --- a/tests/ExcelReader.Tests/Cli/CliTests.cs +++ b/tests/ExcelReader.Tests/Cli/CliTests.cs @@ -651,6 +651,29 @@ private static (int Code, string[] Lines, string Err) Preview( return (code, lines, stderr.ToString()); } + [Fact] + public void Should_StartTheErrorOnItsOwnLine_When_ConvertFailsAfterProgressWasPrinted() + { + using MemoryStream stdout = new(); + StringWriter stderr = new(); + + int code = CliCommands.Convert(Fixture("RealExcel.xlsb"), null, null, null, ',', stdout, stderr, + static _ => throw new IOException("boom")); + + Assert.Equal(1, code); + Assert.Equal(Environment.NewLine + "boom" + Environment.NewLine, stderr.ToString()); + } + + [Fact] + public void Should_NotBufferEveryRequestedRowUpFront_When_RowsIsHuge() + { + (int code, string[] lines, string error) = Preview(CsvFile("id\n1\n2\n3\n"), rows: 2_000_000_000); + + Assert.Equal(0, code); + Assert.Empty(error); + Assert.Equal("shape: (3, 1)", lines[0]); + } + [Fact] public void Should_PrintAPolarsStyleTable_When_RunningPreview() { diff --git a/tests/ExcelReader.Tests/Generator/GeneratorTests.cs b/tests/ExcelReader.Tests/Generator/GeneratorTests.cs index b269b0d7..65e595c4 100644 --- a/tests/ExcelReader.Tests/Generator/GeneratorTests.cs +++ b/tests/ExcelReader.Tests/Generator/GeneratorTests.cs @@ -566,6 +566,187 @@ public async Task GeneratedMapRoundTripsThroughRealXlsxWorkbookAndWriter() Assert.Equal("Alpha", values[7]); } + private const string OverrideSource = """ + using System.IO; + using System.Threading.Tasks; + using ExcelReader.Core.Parser; + using ExcelReader.Core.Reader; + using ExcelReader.Core.Reader.Xlsx; + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsx; + + namespace GeneratorTests.OverrideParity + { + public sealed class UpperCaseConverter : IExcelCellConverter + { + public bool TryConvert(in Cell cell, in ExcelRowContext context, out string value) + { + value = cell.GetString().ToUpperInvariant(); + return true; + } + } + + public class BaseRow + { + [ExcelColumn("Full Name")] + public virtual string Name { get; set; } = ""; + [ExcelIgnore] + public virtual string Secret { get; set; } = ""; + [ExcelRequired] + public virtual string Code { get; set; } = ""; + [ExcelConverter(typeof(UpperCaseConverter))] + public virtual string Tag { get; set; } = ""; + public virtual string Plain { get; set; } = ""; + } + + public class MiddleRow : BaseRow + { + public override string Plain { get; set; } = ""; + [ExcelIgnore] + public override string Code { get; set; } = ""; + } + + [ExcelSerializable] + public partial class Row : BaseRow + { + public override string Name { get; set; } = ""; + public override string Secret { get; set; } = ""; + public override string Code { get; set; } = ""; + public override string Tag { get; set; } = ""; + } + + [ExcelSerializable] + public partial class IgnoredRow : MiddleRow + { + public override string Plain { get; set; } = ""; + [ExcelColumn("Derived Name")] + public override string Name { get; set; } = ""; + } + + public static class TestRunner + { + public static async Task RunAsync() + { + var stream = new MemoryStream(); + await using (XlsxWorkbookWriter wb = XlsxWorkbookWriter.Create(stream, leaveOpen: true)) + { + XlsxSheetWriter sheet = wb.AddSheet("S1"); + XlsxRowWriter header = await sheet.StartRowAsync(); + await using (header.ConfigureAwait(false)) + { + header.Write("Full Name"); + header.Write("Secret"); + header.Write("Code"); + header.Write("Tag"); + } + XlsxRowWriter data = await sheet.StartRowAsync(); + await using (data.ConfigureAwait(false)) + { + data.Write("Ada"); + data.Write("hidden"); + data.Write("C1"); + data.Write("abc"); + } + await sheet.EndAsync(); + await wb.EndAsync(); + } + stream.Position = 0; + await using XlsxWorkbook reader = await Excel.FromXlsxAsync(stream); + Row generated = Single(ExcelParser.Generated().Parse(reader.FirstSheet)); + Row reflected = Single(ExcelParser.FromAttributes().Parse(reader.FirstSheet)); + return Describe(generated) + "/" + Describe(reflected); + } + + private static Row Single(System.Collections.Generic.IEnumerable rows) + { + foreach (Row row in rows) + { + return row; + } + return null; + } + + private static string Describe(Row row) => row.Name + "|" + row.Secret + "|" + row.Code + "|" + row.Tag; + } + } + """; + + [Fact] + public async Task GeneratedMapHonorsParserAttributesInheritedByOverrides() + { + (ImmutableCompilationResult result, ImmutableArray diagnostics, string generated) = RunGeneratorWithSource(OverrideSource); + Assert.Empty(diagnostics); + string ignoredMap = generated[generated.IndexOf("IgnoredRow", StringComparison.Ordinal)..]; + Assert.Contains("\"Derived Name\"", ignoredMap, StringComparison.Ordinal); + Assert.DoesNotContain("\"Code\"", ignoredMap, StringComparison.Ordinal); + (EmitResult Emit, byte[] AssemblyBytes) emitted = result.EmitToBytes(); + Assert.True(emitted.Emit.Success, string.Join(Environment.NewLine, emitted.Emit.Diagnostics.Select(static d => d.ToString()))); + Assembly assembly = Assembly.Load(emitted.AssemblyBytes); + MethodInfo method = assembly.GetType("GeneratorTests.OverrideParity.TestRunner")!.GetMethod("RunAsync")!; + string text = await (Task)method.Invoke(null, null)!; + string[] sides = text.Split('/'); + Assert.Equal("Ada||C1|ABC", sides[0]); + Assert.Equal(sides[1], sides[0]); + } + + [Fact] + public void IgnoredOverrideIsNotBoundThroughTheBaseDeclaration() + { + const string source = """ + using ExcelReader.Core.Parser; + + namespace GeneratorTests.IgnoredOverride + { + public class BaseRow + { + public virtual string? Code { get; set; } + } + + [ExcelSerializable] + public partial class Row : BaseRow + { + [ExcelIgnore] + public override string? Code { get; set; } + public string? Name { get; set; } + } + } + """; + (ImmutableCompilationResult result, ImmutableArray diagnostics, string generated) = RunGeneratorWithSource(source); + Assert.Empty(diagnostics); + Assert.True(result.Emit().Success); + Assert.Contains("\"Name\"", generated, StringComparison.Ordinal); + Assert.DoesNotContain("\"Code\"", generated, StringComparison.Ordinal); + } + + [Fact] + public void IgnoredHidingPropertyIsNotBoundThroughTheBaseDeclaration() + { + const string source = """ + using ExcelReader.Core.Parser; + + namespace GeneratorTests.IgnoredHidingProperty + { + public class BaseRow + { + public string? Code { get; set; } + } + + [ExcelSerializable] + public partial class Row : BaseRow + { + [ExcelIgnore] + public new string? Code { get; set; } + public string? Name { get; set; } + } + } + """; + (ImmutableCompilationResult result, ImmutableArray diagnostics, string generated) = RunGeneratorWithSource(source); + Assert.Empty(diagnostics); + Assert.True(result.Emit().Success); + Assert.Contains("\"Name\"", generated, StringComparison.Ordinal); + Assert.DoesNotContain("\"Code\"", generated, StringComparison.Ordinal); + } + [Fact] public void GenericTypeReportsEXR007() { diff --git a/tests/ExcelReader.Tests/Native/ExportsOutParamTests.cs b/tests/ExcelReader.Tests/Native/ExportsOutParamTests.cs index 760c9dce..13223592 100644 --- a/tests/ExcelReader.Tests/Native/ExportsOutParamTests.cs +++ b/tests/ExcelReader.Tests/Native/ExportsOutParamTests.cs @@ -1,4 +1,7 @@ using ExcelReader.Native; +using ExcelReader.Native.Arrow; +using ExcelReader.Native.Reading; +using ExcelReader.Native.Writer; using ExcelReader.Native.Csv; namespace ExcelReader.Tests.Native @@ -101,5 +104,130 @@ public void RowsOpenOnAnUnknownWorkbookStillZeroesTheCursorOutParam() Assert.Equal(NativeStatus.InvalidHandle, status); Assert.Equal(0, outRows); } + + [Fact] + public void ParseTypedRejectingItsArgumentsStillZeroesTheTableOutParam() + { + NativeTable table; + new Span(&table, sizeof(NativeTable)).Fill(0x5A); + + int status = ((delegate* unmanaged)&Exports.ParseTyped)( + 0, 0, null, 0, 0, 0, &table); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Equal(0, table.ColumnCount); + } + + [Fact] + public void ParseArrowRejectingItsArgumentsStillZeroesBothOutParams() + { + ArrowArray array; + new Span(&array, sizeof(ArrowArray)).Fill(0x5A); + + int status = ((delegate* unmanaged)&Exports.ParseArrow)( + 0, 0, null, 0, 0, 0, &array, null); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Equal(0, array.Length); + } + + [Fact] + public void WriteTypedToMemoryRejectingItsArgumentsStillZeroesTheBufferOutParam() + { + NativeBuffer buffer = new() { Data = Sentinel, Length = Sentinel }; + + int status = ((delegate* unmanaged)&Exports.WriteTypedToMemory)( + 0, null, null, null, &buffer); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Equal(nint.Zero, buffer.Data); + Assert.Equal(0, buffer.Length); + } + + [Fact] + public void EncryptPackageToMemoryRejectingItsArgumentsStillZeroesTheBufferOutParam() + { + NativeBuffer buffer = new() { Data = Sentinel, Length = Sentinel }; + + int status = ((delegate* unmanaged)&Exports.EncryptPackageToMemory)( + null, 0, null, 0, &buffer); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Equal(nint.Zero, buffer.Data); + Assert.Equal(0, buffer.Length); + } + + [Fact] + public void OpenWriteHandleRejectingItsArgumentsStillZeroesTheHandleOutParam() + { + nint outHandle = Sentinel; + + int status = ((delegate* unmanaged)&Exports.OpenWriteHandle)( + null, 0, 0, null, &outHandle); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Equal(0, outHandle); + } + + [Fact] + public void AddStyleRejectingItsArgumentsStillZeroesTheStyleIdOutParam() + { + int outStyleId = Sentinel; + + int status = ((delegate* unmanaged)&Exports.AddStyle)(0, null, &outStyleId); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Equal(0, outStyleId); + } + + [Fact] + public void RowsNextCellsRejectingItsArgumentsStillZeroesTheCountOutParam() + { + int outCount = Sentinel; + + int status = ((delegate* unmanaged)&Exports.RowsNextCells)(0, null, &outCount); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Equal(0, outCount); + } + + [Fact] + public void ParseTypedWithANullColumnNameRejectsItAsInvalidArgument() + { + int nameLen = 0; + byte* name = null; + NativeColumnSpecRaw spec = new() { Names = &name, NameLens = &nameLen, NameCount = 1 }; + NativeTable table; + + int status = ((delegate* unmanaged)&Exports.ParseTyped)( + 0, 0, &spec, 1, 0, 0, &table); + + Assert.Equal(NativeStatus.InvalidArgument, status); + } + + [Fact] + public void OpenMemoryAcceptsANullPointerWithZeroLength() + { + nint outHandle = Sentinel; + + int status = ((delegate* unmanaged)&Exports.OpenMemory)( + null, 0, NativeFormat.Csv, null, &outHandle); + + Assert.Equal(NativeStatus.Ok, status); + Assert.NotEqual(0, outHandle); + ((delegate* unmanaged)&Exports.Close)(outHandle); + } + + [Fact] + public void OpenMemoryRejectingANullPointerWithLengthSetsALastError() + { + nint outHandle = Sentinel; + + int status = ((delegate* unmanaged)&Exports.OpenMemory)( + null, 5, NativeFormat.Csv, null, &outHandle); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.NotEmpty(NativeApi.LastErrorText()); + } } } diff --git a/tests/ExcelReader.Tests/Parser/CsvParserTests.cs b/tests/ExcelReader.Tests/Parser/CsvParserTests.cs index f6c28366..4d9b4645 100644 --- a/tests/ExcelReader.Tests/Parser/CsvParserTests.cs +++ b/tests/ExcelReader.Tests/Parser/CsvParserTests.cs @@ -428,5 +428,118 @@ public void EnumeratorResetThrows() Assert.Throws(e.Reset); } + + [Fact] + public async Task ACsvSheetHeldAsIExcelSheetParsesLikeTheCsvOverload() + { + using var ms = Csv("At,Day\n01/15/2024 10:30:00,01/15/2024\n\n"); + using var reader = Excel.FromCsv(ms); + IExcelSheet sheet = reader.FirstSheet; + + StampRow row = Assert.Single(ExcelParser.FromAttributes().Parse(sheet).ToList()); + StampRow asyncRow = Assert.Single(await ExcelParser.FromAttributes().Parse(sheet).ToListAsync(TestContext.Current.CancellationToken)); + + Assert.Equal(new DateTime(2024, 1, 15, 10, 30, 0), row.At); + Assert.Equal(new DateOnly(2024, 1, 15), row.Day); + Assert.Equal(row.At, asyncRow.At); + } + + [Fact] + public void ACsvSheetHeldAsIExcelSheetSkipsBlankLinesWithoutARequiredFailure() + { + using var ms = Csv("Id,Note\n7,valid\n\n8,more\n\n"); + using var reader = Excel.FromCsv(ms); + IExcelSheet sheet = reader.FirstSheet; + + List rows = ExcelParser.FromAttributes().Parse(sheet).ToList(); + + Assert.Equal([7, 8], rows.Select(r => r.Id)); + } + + [Fact] + public void ACsvSheetHeldAsIExcelSheetRejectsAnIntegerTimeOnly() + { + using var ms = Csv("Start\n1230\n"); + using var reader = Excel.FromCsv(ms); + IExcelSheet sheet = reader.FirstSheet; + var config = new ExcelParserOptions { ThrowOnParseFailure = true }; + + ExcelParseException ex = Assert.Throws( + () => ExcelParser.FromAttributes(config).Parse(sheet).ToList()); + Assert.Equal("Start", ex.ColumnName); + } + + private sealed class IndexedRow + { + private readonly string[] _extra = new string[4]; + + public int Id { get; set; } + + public string this[int index] + { + get + { + return _extra[index]; + } + set + { + _extra[index] = value; + } + } + } + + [Fact] + public void AModelWithAPublicIndexerBindsItsOtherProperties() + { + using var ms = Csv("Id,Item\n7,x\n"); + using var reader = Excel.FromCsv(ms); + + IndexedRow row = Assert.Single(ExcelParser.FromAttributes().Parse(reader.FirstSheet).ToList()); + + Assert.Equal(7, row.Id); + } + + private class CodeBase + { + public int Code { get; set; } + } + + private sealed class HiddenCodeRow : CodeBase + { + public new string? Code { get; set; } + } + + [Fact] + public void APropertyHiddenWithNewBindsTheDerivedOne() + { + using var ms = Csv("Code" + Environment.NewLine + "A1" + Environment.NewLine); + using var reader = Excel.FromCsv(ms); + + HiddenCodeRow row = Assert.Single(ExcelParser.FromAttributes().Parse(reader.FirstSheet).ToList()); + + Assert.Equal("A1", row.Code); + } + + private class VirtualCodeBase + { + public virtual string? Code { get; set; } + } + + private sealed class IgnoredOverrideRow : VirtualCodeBase + { + [ExcelIgnore] + public override string? Code { get; set; } + } + + [Fact] + public void AnIgnoredOverrideIsNotBoundThroughTheBaseDeclaration() + { + using var ms = Csv("Code" + Environment.NewLine + "A1" + Environment.NewLine); + using var reader = Excel.FromCsv(ms); + + IgnoredOverrideRow row = Assert.Single(ExcelParser.FromAttributes().Parse(reader.FirstSheet).ToList()); + + Assert.Null(row.Code); + } } } diff --git a/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvParallelDifferentialTests.cs b/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvParallelDifferentialTests.cs index 4ed4637e..4a1b3c85 100644 --- a/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvParallelDifferentialTests.cs +++ b/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvParallelDifferentialTests.cs @@ -132,5 +132,62 @@ public async Task MatchesSequentialOnALargeGeneratedFile() Assert.Equal(expected, actual); } + + private struct StructRow + { + public string? Name { get; set; } + public int Age { get; set; } + public string? Note { get; set; } + } + + private static byte[] MultiChunkCsv() + { + var sb = new StringBuilder("Name,Age,Note\n"); + for (int i = 0; i < 3_000; i++) + { + string name = i % 5 == 0 ? $"\"multi\nline{i}\"" : $"name{i}"; + sb.Append(CultureInfo.InvariantCulture, $"{name},{i},note{i}\n"); + if (i % 97 == 0) + { + sb.Append('\n'); + } + } + return Encoding.UTF8.GetBytes(sb.ToString()); + } + + private static async Task> ParallelStructAsync(byte[] csv, int dop, int chunkSize) + { + var list = new List(); + await foreach (StructRow row in ParallelCsvFactory.CreateWithChunkSize( + csv.AsMemory(), dop, chunkSize, null, ExcelParser.FromAttributes(), CancellationToken.None)) + { + list.Add($"{row.Name}{row.Age}{row.Note}"); + } + return list; + } + + [Fact] + public async Task StructAndClassModelsMatchSequentialAcrossManyChunks() + { + byte[] csv = MultiChunkCsv(); + List expected = Sequential(csv); + using var reader = Excel.FromCsv(csv); + var expectedStructs = new List(); + foreach (StructRow row in ExcelParser.FromAttributes().Parse(reader.FirstSheet)) + { + expectedStructs.Add($"{row.Name}{row.Age}{row.Note}"); + } + Assert.Equal(3_000, expected.Count); + Assert.Equal(expected, expectedStructs); + + foreach (int dop in new[] { 2, 4, 8 }) + { + foreach (int chunkSize in new[] { 64, 1_000, 4_096, 16_384 }) + { + Assert.Equal(expected, await ParallelAsync(csv, dop, chunkSize)); + Assert.Equal(expected, await ParallelStructAsync(csv, dop, chunkSize)); + } + } + } } } diff --git a/tests/ExcelReader.Tests/Parser/ParallelCsv/ParallelCsvEnumerableTests.cs b/tests/ExcelReader.Tests/Parser/ParallelCsv/ParallelCsvEnumerableTests.cs index 4050c2d6..f7d1c997 100644 --- a/tests/ExcelReader.Tests/Parser/ParallelCsv/ParallelCsvEnumerableTests.cs +++ b/tests/ExcelReader.Tests/Parser/ParallelCsv/ParallelCsvEnumerableTests.cs @@ -221,5 +221,58 @@ public async Task ProducesTheSameSequenceAtEveryDegreeOfParallelism() Assert.Equal(baseline, actual); } } + + [Fact] + public async Task AParseFailureYieldsTheSequentialRowsBeforeItAndThenEnds() + { + var sb = new StringBuilder("Name,Age\n"); + for (int i = 0; i < 400; i++) + { + string age = i == 250 ? "not-a-number" : i.ToString(CultureInfo.InvariantCulture); + sb.Append(CultureInfo.InvariantCulture, $"name{i:D5},{age}\n"); + } + byte[] csv = Encoding.UTF8.GetBytes(sb.ToString()); + var config = new ExcelParserOptions { ThrowOnParseFailure = true }; + + using CsvReader sequentialReader = Excel.FromCsv(csv); + int sequentialRows = 0; + ExcelParseException expected = Assert.Throws(() => + { + foreach (Row row in ExcelParser.FromAttributes(config).Parse(sequentialReader.FirstSheet)) + { + sequentialRows++; + } + }); + + IAsyncEnumerator enumerator = Build(csv, dop: 4, chunkSizeOverride: 64, config) + .GetAsyncEnumerator(TestContext.Current.CancellationToken); + int parallelRows = 0; + ExcelParseException actual = await Assert.ThrowsAsync(async () => + { + while (await enumerator.MoveNextAsync()) + { + parallelRows++; + } + }); + + Assert.Equal(expected.Row, actual.Row); + Assert.Equal(expected.Message, actual.Message); + Assert.Equal(250, sequentialRows); + Assert.Equal(sequentialRows, parallelRows); + Assert.False(await enumerator.MoveNextAsync()); + await enumerator.DisposeAsync(); + await enumerator.DisposeAsync(); + } + + [Fact] + public async Task DisposingBeforeTheFirstMoveNextIsSafe() + { + IAsyncEnumerator enumerator = Build(BuildCsv(500), dop: 4, chunkSizeOverride: 64) + .GetAsyncEnumerator(TestContext.Current.CancellationToken); + + await enumerator.DisposeAsync(); + + Assert.False(await enumerator.MoveNextAsync()); + } } } diff --git a/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs b/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs index 0fc07212..221e9801 100644 --- a/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs +++ b/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs @@ -400,5 +400,166 @@ public async Task ANonSeekableStreamFallsBackWithoutThrowing() Assert.Equal(20_000, actual.Count); Assert.Equal(19_999, actual[^1].Age); } + + [Fact] + public async Task APartitionedParseHonorsTheTokenPassedToParseAsync() + { + using CsvReader source = Excel.FromCsv(BuildCsv(20_000)); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + IAsyncEnumerable rows = SheetParallel.ParseAsync(source.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: cts.Token); + + await Assert.ThrowsAnyAsync(() => DrainAsync(rows)); + } + + [Fact] + public async Task ASequentialParseKeepsTheParseAsyncTokenWhenEnumeratedWithAnother() + { + using CsvReader source = Excel.FromCsv(BuildCsv(3)); + using var cts = new CancellationTokenSource(); + using var other = new CancellationTokenSource(); + await cts.CancelAsync(); + + IAsyncEnumerable rows = SheetParallel.ParseAsync(source.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 1 }, ct: cts.Token); + + await Assert.ThrowsAnyAsync(async () => await rows.ToListAsync(other.Token)); + } + + [Fact] + public async Task TheSequentialFallbackAfterAHugeHeaderCanBeEnumeratedTwice() + { + byte[] csv = Encoding.UTF8.GetBytes("Name,Age," + new string('x', 300_000) + "\nada,1\ngrace,2\n"); + using CsvReader source = Excel.FromCsv(csv); + IAsyncEnumerable rows = SheetParallel.ParseAsync(source.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + + Assert.Equal(2, (await DrainAsync(rows)).Count); + Assert.Equal(2, (await DrainAsync(rows)).Count); + } + + [Fact] + public async Task ASourceShorterThanTheHeaderRowYieldsNothingLikeTheSequentialParser() + { + byte[] csv = Encoding.UTF8.GetBytes("Name,Age\n" + new string('x', 300_000) + "\n"); + var parser = ExcelParser.FromAttributes(new ExcelParserOptions { HeaderRow = 5 }); + using CsvReader sequential = Excel.FromCsv(csv); + Assert.Empty(parser.Parse(sequential.FirstSheet).ToList()); + + using CsvReader source = Excel.FromCsv(csv); + List actual = await DrainAsync(SheetParallel.ParseAsync(source.FirstSheet, parser, options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken)); + + Assert.Empty(actual); + } + + [Fact] + public async Task BreakingEarlyLetsTheResultBeEnumeratedAgainAndStillClosesTheFile() + { + string path = Path.Combine(Path.GetTempPath(), $"exr-{Guid.NewGuid():N}.csv"); + await File.WriteAllBytesAsync(path, BuildCsv(40_000), TestContext.Current.CancellationToken); + try + { + using (CsvReader source = Excel.FromCsvFile(path)) + { + IAsyncEnumerable rows = SheetParallel.ParseAsync(source.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + int taken = 0; + await foreach (Row row in rows) + { + taken++; + if (taken == 10) + { + break; + } + } + + Assert.Equal(10, taken); + Assert.Equal(40_000, (await DrainAsync(rows).WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)).Count); + } + + await using var exclusive = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None); + Assert.True(exclusive.Length > 0); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public async Task DisposingAnEnumeratorTwiceReleasesTheReaderOnlyOnce() + { + using CsvReader source = Excel.FromCsv(BuildCsv(20_000)); + IAsyncEnumerable rows = SheetParallel.ParseAsync(source.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + + IAsyncEnumerator enumerator = rows.GetAsyncEnumerator(TestContext.Current.CancellationToken); + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal("name00000", enumerator.Current.Name); + await enumerator.DisposeAsync(); + await enumerator.DisposeAsync(); + + Assert.False(await enumerator.MoveNextAsync()); + Assert.Equal(20_000, (await DrainAsync(rows)).Count); + } + + [Fact] + public async Task AConverterThrowingOnARealRowSurfacesTheSequentialExceptionAfterTheSameRows() + { + var sb = new StringBuilder("Name,Age\n"); + for (int i = 0; i < 40_000; i++) + { + sb.Append(i == 30_000 ? "bad,BOOM\n" : $"name{i:D5},{i}\n"); + } + byte[] csv = Encoding.UTF8.GetBytes(sb.ToString()); + + using CsvReader sequentialSource = Excel.FromCsv(csv); + int sequentialRows = 0; + InvalidOperationException sequential = Assert.Throws(() => + { + foreach (StrictRow row in ExcelParser.FromAttributes().Parse(sequentialSource.FirstSheet)) + { + sequentialRows++; + } + }); + + using CsvReader parallelSource = Excel.FromCsv(csv); + IAsyncEnumerator enumerator = SheetParallel.ParseAsync(parallelSource.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken) + .GetAsyncEnumerator(TestContext.Current.CancellationToken); + int parallelRows = 0; + InvalidOperationException parallel = await Assert.ThrowsAsync(async () => + { + while (await enumerator.MoveNextAsync()) + { + parallelRows++; + } + }); + + Assert.Equal(sequential.Message, parallel.Message); + Assert.Equal(30_000, sequentialRows); + Assert.Equal(sequentialRows, parallelRows); + Assert.False(await enumerator.MoveNextAsync()); + await enumerator.DisposeAsync(); + } + + [Fact] + public async Task CancellingTheParseAsyncTokenMidEnumerationThrows() + { + using CsvReader source = Excel.FromCsv(BuildCsv(40_000)); + using var cts = new CancellationTokenSource(); + IAsyncEnumerable rows = SheetParallel.ParseAsync(source.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: cts.Token); + int taken = 0; + + await Assert.ThrowsAnyAsync(async () => + { + await foreach (Row row in rows) + { + taken++; + if (taken == 5) + { + await cts.CancelAsync(); + } + } + }); + + Assert.InRange(taken, 5, 39_999); + } } } diff --git a/tests/ExcelReader.Tests/Parser/ParserFeatureTests.cs b/tests/ExcelReader.Tests/Parser/ParserFeatureTests.cs index 00d4e276..7324379f 100644 --- a/tests/ExcelReader.Tests/Parser/ParserFeatureTests.cs +++ b/tests/ExcelReader.Tests/Parser/ParserFeatureTests.cs @@ -80,6 +80,21 @@ public async Task PtBrCultureParsesCommaDecimalAndThousandsSeparator() Assert.Equal(1234.56m, row.Amount); } + [Theory] + [InlineData("pt-BR")] + [InlineData("fr-FR")] + public void CultureDoesNotApplyToWorkbookStoredNumbers(string culture) + { + using var ms = WorkbookBuilder.Build( + """Amount35.840000000000003"""); + using var reader = Excel.FromXlsx(ms); + var config = new ExcelParserOptions { Culture = CultureInfo.GetCultureInfo(culture) }; + + MoneyRow row = ExcelParser.FromAttributes(config).Parse(reader.FirstSheet).Single(); + + Assert.Equal(35.840000000000003m, row.Amount); + } + [Fact] public async Task InvariantCultureRejectsCommaDecimal() { diff --git a/tests/ExcelReader.Tests/Reader/BlankCellTypedTests.cs b/tests/ExcelReader.Tests/Reader/BlankCellTypedTests.cs new file mode 100644 index 00000000..72e2a805 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/BlankCellTypedTests.cs @@ -0,0 +1,294 @@ +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using ExcelReader.Core.Parser; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Columnar; +using ExcelReader.Core.Reader.Csv; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Reader.Schema; + +namespace ExcelReader.Tests.Reader +{ + public sealed class BlankCellTypedTests + { + private static readonly DateTime Sample = new(2024, 3, 5, 12, 0, 0); + + private sealed class BlankRow + { + public int? Qty { get; set; } + public double? Rate { get; set; } + public DateTime? When { get; set; } + public DateOnly? Day { get; set; } + public decimal? Amount { get; set; } + public bool? Flag { get; set; } + } + + private readonly struct AcceptsEmpty : IUtf8SpanParsable + { + public static AcceptsEmpty Parse(ReadOnlySpan utf8Text, IFormatProvider? provider) + { + return default; + } + + public static bool TryParse(ReadOnlySpan utf8Text, IFormatProvider? provider, out AcceptsEmpty result) + { + result = default; + return true; + } + } + + public static TheoryData Cases() + { + TheoryData data = []; + foreach (ExcelColumnType type in Enum.GetValues()) + { + foreach (bool nullable in new[] { false, true }) + { + foreach (bool csv in new[] { false, true }) + { + data.Add(type, nullable, csv, 1); + } + } + } + data.Add(ExcelColumnType.Int64Column, true, true, 4); + data.Add(ExcelColumnType.TimestampColumn, true, false, 4); + return data; + } + + [Theory] + [MemberData(nameof(Cases))] + public void Blank_And_Missing_Cells_Append_Per_Column_Nullability(ExcelColumnType type, bool nullable, bool csv, int dop) + { + using IExcelWorkbook workbook = csv ? OpenCsv(type) : OpenXlsx(type); + TypedColumnSpec[] specs = + [ + new() { Names = ["k"], Type = ExcelColumnType.Int64Column }, + new() { Names = ["v"], Type = type, Nullable = nullable }, + ]; + bool text = type is ExcelColumnType.StringColumn or ExcelColumnType.StringDictionaryColumn; + if (!nullable && !text) + { + InvalidOperationException error = Assert.Throws(() => + TypedTableParser.Parse(workbook.FirstSheet, specs, 1, dop, allowDictionary: true, 0, out _, out _)); + Assert.Equal("column 1 (\"v\") has a value that failed to convert and is not nullable.", error.Message); + return; + } + using ColumnarTable? table = TypedTableParser.Parse(workbook.FirstSheet, specs, 1, dop, allowDictionary: true, 0, out _, out string? specError); + Assert.Null(specError); + Assert.NotNull(table); + Assert.Equal(4, table.RowCount); + Assert.Equal(["v:1", "v:2", "v:3", "v:4"], Render(table.Columns[0], ExcelColumnType.Int64Column)); + string valid = Expected(type); + string[] expected = nullable + ? [$"v:{valid}", $"n:{Raw(type)}", $"v:{valid}", $"n:{Raw(type)}"] + : [$"v:{valid}", "v:", $"v:{valid}", "v:"]; + Assert.Equal(expected, Render(table.Columns[1], type)); + Assert.Equal(nullable, table.Columns[1].Validity != IntPtr.Zero); + } + + [Fact] + public void Nullable_Columns_Keep_Appending_After_A_Blank() + { + using IExcelWorkbook workbook = Excel.FromCsv(Encoding.UTF8.GetBytes("a,b,c\n,,\n1,2.5,x\n,,\n")); + TypedColumnSpec[] specs = + [ + new() { Names = ["a"], Type = ExcelColumnType.Int64Column, Nullable = true }, + new() { Names = ["b"], Type = ExcelColumnType.Float64Column, Nullable = true }, + new() { Names = ["c"], Type = ExcelColumnType.StringColumn, Nullable = true }, + ]; + using ColumnarTable table = TypedTableParser.Parse(workbook.FirstSheet, specs, 1, 1, allowDictionary: false, 0, out _, out _)!; + Assert.Equal(3, table.RowCount); + Assert.Equal(["n:0", "v:1", "n:0"], Render(table.Columns[0], ExcelColumnType.Int64Column)); + Assert.Equal(["n:0", "v:2.5", "n:0"], Render(table.Columns[1], ExcelColumnType.Float64Column)); + Assert.Equal(["n:", "v:x", "n:"], Render(table.Columns[2], ExcelColumnType.StringColumn)); + } + + [Fact] + public void Csv_Model_Nullable_Properties_Are_Null_For_Blank_And_Missing_Fields() + { + using CsvReader reader = Excel.FromCsv(Encoding.UTF8.GetBytes( + "Qty,Rate,When,Day,Amount,Flag\n1,2.5,2024-03-05T12:00:00,2024-03-05,9.99,true\n2,,,,,\n,4.5,,,10,\n5,1.5\n")); + List rows = [.. ExcelParser.FromAttributes().Parse(reader.FirstSheet)]; + AssertModel(rows); + } + + [Fact] + public async Task Xlsx_Model_Nullable_Properties_Are_Null_For_Blank_And_Missing_Cells() + { + await using MemoryStream stream = await TypedWorkbook.BuildAsync( + ["Qty", "Rate", "When", "Day", "Amount", "Flag"], + [1, 2.5, Sample, Sample, 9.99m, true], + [2, null, null, null, null, null], + [new Gap(), 4.5, new Gap(), new Gap(), 10m], + [5, 1.5]); + await using var reader = Excel.FromXlsx(stream); + List rows = [.. ExcelParser.FromAttributes().Parse(reader.FirstSheet)]; + AssertModel(rows); + } + + private static void AssertModel(List rows) + { + Assert.Equal(4, rows.Count); + Assert.Equal((1, 2.5, (DateTime?)Sample, (DateOnly?)DateOnly.FromDateTime(Sample), (decimal?)9.99m, (bool?)true), + (rows[0].Qty, rows[0].Rate, rows[0].When, rows[0].Day, rows[0].Amount, rows[0].Flag)); + Assert.Equal((2, null, null, null, null, null), (rows[1].Qty, rows[1].Rate, rows[1].When, rows[1].Day, rows[1].Amount, rows[1].Flag)); + Assert.Equal((null, 4.5, null, null, (decimal?)10m, null), (rows[2].Qty, rows[2].Rate, rows[2].When, rows[2].Day, rows[2].Amount, rows[2].Flag)); + Assert.Equal((5, 1.5, null, null, null, null), (rows[3].Qty, rows[3].Rate, rows[3].When, rows[3].Day, rows[3].Amount, rows[3].Flag)); + } + + [Fact] + public void Empty_Cell_Reads_Fail_For_Dates_And_Parsable_Primitives() + { + AssertBlankFails(new Cell(CellType.Empty, default)); + AssertBlankFails(new Cell(CellType.ExcelString, default)); + } + + private static void AssertBlankFails(Cell cell) + { + Assert.False(cell.TryGetDateTime(isDate1904: false, out DateTime date)); + Assert.Equal(default, date); + Assert.False(cell.TryParse(null, out double d)); + Assert.False(cell.TryParse(null, out float f)); + Assert.False(cell.TryParse(null, out Half h)); + Assert.False(cell.TryParse(null, out decimal m)); + Assert.False(cell.TryParse(null, out int i)); + Assert.False(cell.TryParse(null, out long l)); + Assert.False(cell.TryParse(null, out byte b)); + Assert.False(cell.TryParse(null, out Guid guid)); + Assert.Equal((0d, 0f, Half.Zero, 0m, 0, 0L, (byte)0, Guid.Empty), (d, f, h, m, i, l, b, guid)); + } + + [Fact] + public void Empty_Cell_Read_Still_Reaches_A_Type_That_Accepts_It() + { + Cell cell = new(CellType.Empty, default); + Assert.True(cell.TryParse(null, out AcceptsEmpty _)); + } + + [Fact] + public void Number_Cells_With_Empty_Text_Still_Read() + { + Cell cell = new(CellType.Number, default, 45000, hasNumber: true, styleIndex: 0); + Assert.True(cell.TryGetDateTime(isDate1904: false, out DateTime date)); + Assert.Equal(DateTime.FromOADate(45000), date); + Assert.True(cell.TryParse(null, out int value)); + Assert.Equal(45000, value); + } + + private static CsvReader OpenCsv(ExcelColumnType type) + { + string sample = type switch + { + ExcelColumnType.Int64Column => "7", + ExcelColumnType.Float64Column => "1.5", + ExcelColumnType.BoolColumn => "true", + ExcelColumnType.DateColumn => "2024-03-05", + ExcelColumnType.TimeColumn => "12:00:00", + ExcelColumnType.TimestampColumn => "2024-03-05T12:00:00", + _ => "abc", + }; + return Excel.FromCsv(Encoding.UTF8.GetBytes($"k,v\n1,{sample}\n2,\n3,{sample}\n4\n")); + } + + private static XlsxWorkbook OpenXlsx(ExcelColumnType type) + { + object sample = type switch + { + ExcelColumnType.Int64Column => 7, + ExcelColumnType.Float64Column => 1.5, + ExcelColumnType.BoolColumn => true, + ExcelColumnType.DateColumn or ExcelColumnType.TimeColumn or ExcelColumnType.TimestampColumn => Sample, + _ => "abc", + }; + MemoryStream stream = TypedWorkbook.BuildAsync(["k", "v"], [1, sample], [2, null], [3, sample], [4]).GetAwaiter().GetResult(); + return Excel.FromXlsx(stream, leaveOpen: false); + } + + private static string Expected(ExcelColumnType type) + { + return type switch + { + ExcelColumnType.Int64Column => "7", + ExcelColumnType.Float64Column => "1.5", + ExcelColumnType.BoolColumn => "1", + ExcelColumnType.DateColumn => (DateOnly.FromDateTime(Sample).DayNumber - DateOnly.FromDateTime(DateTime.UnixEpoch).DayNumber).ToString(CultureInfo.InvariantCulture), + ExcelColumnType.TimeColumn => (Sample.TimeOfDay.Ticks / 10).ToString(CultureInfo.InvariantCulture), + ExcelColumnType.TimestampColumn => ((Sample - DateTime.UnixEpoch).Ticks / 10).ToString(CultureInfo.InvariantCulture), + _ => "abc", + }; + } + + private static string Raw(ExcelColumnType type) + { + return type switch + { + ExcelColumnType.StringColumn => "", + ExcelColumnType.StringDictionaryColumn => "abc", + _ => "0", + }; + } + + private static string[] Render(BuiltColumn column, ExcelColumnType type) + { + int count = (int)column.Length; + byte[] validity = new byte[(count + 7) / 8]; + if (column.Validity != IntPtr.Zero) + { + Marshal.Copy(column.Validity, validity, 0, validity.Length); + } + string[] raw = RawValues(column, type, count); + string[] rendered = new string[count]; + for (int i = 0; i < count; i++) + { + bool valid = column.Validity == IntPtr.Zero || (validity[i >> 3] & (1 << (i & 7))) != 0; + rendered[i] = (valid ? "v:" : "n:") + raw[i]; + } + return rendered; + } + + private static string[] RawValues(BuiltColumn column, ExcelColumnType type, int count) + { + switch (type) + { + case ExcelColumnType.StringColumn: + return Strings(column.Values, column.Data, (int)column.DataLen, count); + case ExcelColumnType.StringDictionaryColumn: + int[] indices = new int[count]; + Marshal.Copy(column.Values, indices, 0, count); + string[] entries = Strings(column.DictionaryOffsets, column.Data, (int)column.DataLen, (int)column.DictionaryLength); + return [.. indices.Select(i => entries[i])]; + case ExcelColumnType.Float64Column: + double[] doubles = new double[count]; + Marshal.Copy(column.Values, doubles, 0, count); + return [.. doubles.Select(d => d.ToString(CultureInfo.InvariantCulture))]; + case ExcelColumnType.BoolColumn: + byte[] bytes = new byte[count]; + Marshal.Copy(column.Values, bytes, 0, count); + return [.. bytes.Select(b => b.ToString(CultureInfo.InvariantCulture))]; + case ExcelColumnType.DateColumn: + int[] ints = new int[count]; + Marshal.Copy(column.Values, ints, 0, count); + return [.. ints.Select(i => i.ToString(CultureInfo.InvariantCulture))]; + default: + long[] longs = new long[count]; + Marshal.Copy(column.Values, longs, 0, count); + return [.. longs.Select(l => l.ToString(CultureInfo.InvariantCulture))]; + } + } + + private static string[] Strings(IntPtr offsetsBlock, IntPtr dataBlock, int dataLength, int count) + { + int[] offsets = new int[count + 1]; + Marshal.Copy(offsetsBlock, offsets, 0, offsets.Length); + byte[] data = new byte[dataLength]; + Marshal.Copy(dataBlock, data, 0, dataLength); + string[] strings = new string[count]; + for (int i = 0; i < count; i++) + { + strings[i] = Encoding.UTF8.GetString(data, offsets[i], offsets[i + 1] - offsets[i]); + } + return strings; + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/CellNullProviderTests.cs b/tests/ExcelReader.Tests/Reader/CellNullProviderTests.cs index 16b881cb..2deb6d5b 100644 --- a/tests/ExcelReader.Tests/Reader/CellNullProviderTests.cs +++ b/tests/ExcelReader.Tests/Reader/CellNullProviderTests.cs @@ -43,6 +43,25 @@ public void NullProviderParsesDecimalsInvariantlyWhateverTheCurrentCulture() Assert.Equal(35.840000000000003m, ParseUnderBrazilianCulture(cell)); } + [Theory] + [InlineData("pt-BR", CellType.Number)] + [InlineData("fr-FR", CellType.Number)] + [InlineData("de-DE", CellType.Date)] + public void WorkbookNumberTextParsesInvariantlyWhateverTheProvider(string culture, CellType type) + { + var provider = CultureInfo.GetCultureInfo(culture); + var cell = new Cell(type, "35.840000000000003"u8); + + Assert.True(cell.TryParse(provider, out double d)); + Assert.Equal(35.840000000000003, d); + Assert.True(cell.TryParse(provider, out decimal m)); + Assert.Equal(35.840000000000003m, m); + Assert.True(cell.TryParse(provider, out float f)); + Assert.Equal(35.84f, f); + Assert.True(cell.TryParse(provider, out Half h)); + Assert.Equal((Half)35.84, h); + } + [Fact] public void StoredNumberFallbackParsesItsInvariantTextWhateverTheProvider() { diff --git a/tests/ExcelReader.Tests/Reader/DetectFileFormatContractTests.cs b/tests/ExcelReader.Tests/Reader/DetectFileFormatContractTests.cs index e72aaf7f..7d77d7cd 100644 --- a/tests/ExcelReader.Tests/Reader/DetectFileFormatContractTests.cs +++ b/tests/ExcelReader.Tests/Reader/DetectFileFormatContractTests.cs @@ -18,6 +18,31 @@ private static byte[] WordLikeZip() return buffer.ToArray(); } + private static byte[] EncryptedZip() + { + byte[] bytes = WordLikeZip(); + const int LocalFlagsOffset = 6; + const int CentralFlagsOffset = 8; + int central = bytes.AsSpan().IndexOf("PK\u0001\u0002"u8); + Assert.True(central > 0); + bytes[LocalFlagsOffset] |= 1; + bytes[central + CentralFlagsOffset] |= 1; + return bytes; + } + + private static byte[] ZipWithMoreEntriesThanTheDefaultLimit() + { + using MemoryStream buffer = new(); + using (ZipArchive zip = new(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + for (int i = 0; i <= ExcelReaderOptions.Default.MaxZipEntries; i++) + { + zip.CreateEntry(i.ToString(System.Globalization.CultureInfo.InvariantCulture), CompressionLevel.NoCompression); + } + } + return buffer.ToArray(); + } + public static TheoryData NotAWorkbook => [ ZipSignatureOnly, @@ -39,6 +64,30 @@ public async Task ReturnsUnknownForAZipThatIsNotAWorkbook(byte[] bytes) Assert.Equal(ExcelFileFormat.Unknown, Excel.DetectFileFormat(bytes)); } + [Fact] + public Task ReturnsUnknownForAnEncryptedZip() + { + return AssertUnknownAsync(EncryptedZip()); + } + + [Fact] + public Task ReturnsUnknownForAZipPastTheEntryLimit() + { + return AssertUnknownAsync(ZipWithMoreEntriesThanTheDefaultLimit()); + } + + private static async Task AssertUnknownAsync(byte[] bytes) + { + using MemoryStream stream = new(bytes); + await using MemoryStream asyncStream = new(bytes); + + Assert.Equal(ExcelFileFormat.Unknown, Excel.DetectFileFormat(stream)); + Assert.Equal(0, stream.Position); + Assert.Equal(ExcelFileFormat.Unknown, await Excel.DetectFileFormatAsync(asyncStream, TestContext.Current.CancellationToken)); + Assert.Equal(0, asyncStream.Position); + Assert.Equal(ExcelFileFormat.Unknown, Excel.DetectFileFormat(bytes)); + } + [Fact] public void OpeningAZipThatIsNotAWorkbookIsRejected() { diff --git a/tests/ExcelReader.Tests/Reader/Internal/StreamSpoolerTests.cs b/tests/ExcelReader.Tests/Reader/Internal/StreamSpoolerTests.cs new file mode 100644 index 00000000..9bd29964 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/StreamSpoolerTests.cs @@ -0,0 +1,171 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Writer.Xls; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class StreamSpoolerTests + { + private const int First = 16; + private const int Max = 64; + + private static byte[] Bytes(int length) + { + byte[] bytes = new byte[length]; + System.Security.Cryptography.RandomNumberGenerator.Fill(bytes); + return bytes; + } + + public static TheoryData Lengths => [0, 1, First - 1, First, First + 1, First + Max, First + Max + 1, 1000]; + + [Theory] + [MemberData(nameof(Lengths))] + public void Spool_Returns_Exact_Bytes(int length) + { + byte[] bytes = Bytes(length); + Assert.Equal(bytes, StreamSpooler.Spool(new NonSeekableStream(bytes), First, Max)); + } + + [Theory] + [MemberData(nameof(Lengths))] + public async Task SpoolAsync_Returns_Exact_Bytes(int length) + { + byte[] bytes = Bytes(length); + Assert.Equal(bytes, await StreamSpooler.SpoolAsync(new TrickleStream(bytes), TestContext.Current.CancellationToken, First, Max)); + } + + [Fact] + public void Spool_Propagates_Read_Failure() + { + NonSeekableStream stream = new(Bytes(100)) { OnRead = () => throw new IOException("boom") }; + Assert.Throws(() => StreamSpooler.Spool(stream, First, Max)); + } + + [Fact] + public async Task SpoolAsync_Throws_When_Cancelled() + { + using CancellationTokenSource cts = new(); + await cts.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => + await StreamSpooler.SpoolAsync(new NonSeekableStream(Bytes(100)), cts.Token, First, Max)); + } + + private static List Dump(IExcelWorkbook reader) + { + List lines = []; + for (int s = 0; s < reader.SheetCount; s++) + { + using IExcelRowEnumerator e = reader.SheetAt(s).GetEnumerator(); + while (e.MoveNext()) + { + lines.Add(string.Join("|", Enumerable.Range(0, e.Current.ColumnCount).Select(c => e.Current[c].GetString()))); + } + } + return lines; + } + + private static string Text(int r) + { + return Guid.NewGuid().ToString("N") + r.ToString(CultureInfo.InvariantCulture); + } + + private static byte[] BuildXlsx() + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions())) + { + XlsxSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < 12000; r++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write(Text(r)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static byte[] BuildXlsb() + { + using MemoryStream buffer = new(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(buffer, leaveOpen: true, new XlsbWriterOptions())) + { + XlsbSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < 12000; r++) + { + using XlsbRowWriter row = sheet.StartRow(); + row.Write(Text(r)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static byte[] BuildXls() + { + using MemoryStream buffer = new(); + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(buffer, leaveOpen: true)) + { + XlsSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < 8000; r++) + { + using XlsRowWriter row = sheet.StartRow(); + row.Write(Text(r)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + [Fact] + public async Task Large_Xlsx_Reads_The_Same_Through_A_NonSeekable_Stream() + { + byte[] bytes = BuildXlsx(); + Assert.True(bytes.Length > StreamSpooler.FirstChunk); + using IExcelWorkbook expected = Excel.FromXlsx(bytes); + using IExcelWorkbook sync = Excel.FromXlsx(new NonSeekableStream(bytes), leaveOpen: false); + using IExcelWorkbook async = await Excel.FromXlsxAsync(new NonSeekableStream(bytes), leaveOpen: false, ct: TestContext.Current.CancellationToken); + List rows = Dump(expected); + Assert.Equal(rows, Dump(sync)); + Assert.Equal(rows, Dump(async)); + } + + [Fact] + public async Task Large_Xlsb_Reads_The_Same_Through_A_NonSeekable_Stream() + { + byte[] bytes = BuildXlsb(); + Assert.True(bytes.Length > StreamSpooler.FirstChunk); + using IExcelWorkbook expected = Excel.FromXlsb(bytes); + using IExcelWorkbook sync = Excel.FromXlsb(new NonSeekableStream(bytes), leaveOpen: false); + using IExcelWorkbook async = await Excel.FromXlsbAsync(new NonSeekableStream(bytes), leaveOpen: false, ct: TestContext.Current.CancellationToken); + List rows = Dump(expected); + Assert.Equal(rows, Dump(sync)); + Assert.Equal(rows, Dump(async)); + } + + [Fact] + public async Task Large_Xls_Reads_The_Same_Through_A_NonSeekable_Stream() + { + byte[] bytes = BuildXls(); + Assert.True(bytes.Length > StreamSpooler.FirstChunk); + using IExcelWorkbook expected = Excel.FromXls(bytes); + using IExcelWorkbook sync = Excel.FromXls(new NonSeekableStream(bytes), leaveOpen: false); + using IExcelWorkbook async = await Excel.FromXlsAsync(new NonSeekableStream(bytes), leaveOpen: false, ct: TestContext.Current.CancellationToken); + List rows = Dump(expected); + Assert.Equal(rows, Dump(sync)); + Assert.Equal(rows, Dump(async)); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/MidStreamCancellationTests.cs b/tests/ExcelReader.Tests/Reader/MidStreamCancellationTests.cs index 81f6e4d2..3ad2ca1d 100644 --- a/tests/ExcelReader.Tests/Reader/MidStreamCancellationTests.cs +++ b/tests/ExcelReader.Tests/Reader/MidStreamCancellationTests.cs @@ -131,6 +131,42 @@ public async Task XlsbCancellationAfterCancelThrowsOnNextMoveNext() await Assert.ThrowsAnyAsync(async () => await e.MoveNextAsync()); } + [Fact] + public async Task XlsxMemoryBackedEnumeratorHonorsCancellation() + { + using MemoryStream built = WorkbookBuilder.Build("1"); + using CancellationTokenSource cts = new(); + cts.Cancel(); + + await using XlsxWorkbook fromMemory = Excel.FromXlsx(built.ToArray()); + await using XlsxWorkbook fromWritableStream = await Excel.FromXlsxAsync(new MemoryStream(built.ToArray()), ct: TestContext.Current.CancellationToken); + + await AssertCancelsAsync(fromMemory.FirstSheet.GetAsyncEnumerator(cts.Token)); + await AssertCancelsAsync(fromWritableStream.FirstSheet.GetAsyncEnumerator(cts.Token)); + } + + [Fact] + public async Task XlsbMemoryBackedEnumeratorHonorsCancellation() + { + byte[] bytes = await BuildSmallXlsbAsync(TestContext.Current.CancellationToken); + using CancellationTokenSource cts = new(); + cts.Cancel(); + + await using XlsbWorkbook fromMemory = Excel.FromXlsb(bytes); + await using XlsbWorkbook fromWritableStream = await Excel.FromXlsbAsync(new MemoryStream(bytes), ct: TestContext.Current.CancellationToken); + + await AssertCancelsAsync(fromMemory.FirstSheet.GetAsyncEnumerator(cts.Token)); + await AssertCancelsAsync(fromWritableStream.FirstSheet.GetAsyncEnumerator(cts.Token)); + } + + private static async Task AssertCancelsAsync(IExcelRowEnumerator e) + { + await using (e) + { + await Assert.ThrowsAnyAsync(async () => await e.MoveNextAsync()); + } + } + private static byte[] BuildLargeXlsx() { StringBuilder sb = new(512 * 1024); diff --git a/tests/ExcelReader.Tests/Reader/SheetDataReaderTests.cs b/tests/ExcelReader.Tests/Reader/SheetDataReaderTests.cs index 36537d69..a67c44ee 100644 --- a/tests/ExcelReader.Tests/Reader/SheetDataReaderTests.cs +++ b/tests/ExcelReader.Tests/Reader/SheetDataReaderTests.cs @@ -353,6 +353,28 @@ public async Task GetFieldTypeFollowsTheCurrentRowsOwnCellTypes() Assert.Equal("Double", data.GetDataTypeName(1)); } + [Fact] + public void GetFieldTypeMatchesGetValueForCellsWhoseValueCannotBeRead() + { + const string styles = + """"""; + using var ms = WorkbookBuilder.Build( + """300000045000abcx""", + styles: styles); + using var reader = Excel.FromXlsx(ms); + using var data = new SheetDataReader(reader.FirstSheet, new SheetDataReaderOptions { HeaderRow = 0 }); + + DataTable schema = data.GetSchemaTable(); + Assert.True(data.Read()); + + Assert.Equal([typeof(string), typeof(DateTime), typeof(string), typeof(string)], + Enumerable.Range(0, 4).Select(i => data.GetValue(i).GetType())); + Assert.Equal([typeof(string), typeof(DateTime), typeof(string), typeof(string)], + Enumerable.Range(0, 4).Select(data.GetFieldType)); + Assert.Equal([typeof(string), typeof(DateTime), typeof(string), typeof(string)], + schema.Rows.Cast().Select(r => (Type)r[SchemaTableColumn.DataType])); + } + [Fact] public async Task IndexersAndTheFixedAdoMembersReportTheDocumentedValues() { diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs index 24765dad..f28b37dd 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs @@ -1,3 +1,5 @@ +using System.IO.Compression; +using System.Text; using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Xlsx; @@ -5,6 +7,7 @@ namespace ExcelReader.Tests.Reader.Xlsx { public class NamespacePrefixAndIsoDateTests { + private const string Main = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; [Fact] public void PrefixedNumberAndInlineStringCellsAreRead() @@ -245,5 +248,91 @@ public void IsoDateCellCombinedWithPrefix() Assert.True(e.Current[0].TryGetDateTime(reader.IsDate1904, out DateTime dt)); Assert.Equal(new DateTime(2026, 3, 4), dt); } + + [Theory] + [InlineData("10:30:00", false)] + [InlineData("10:30:00", true)] + [InlineData("10:30:00-03:00", false)] + [InlineData("10:30:00-03:00", true)] + [InlineData("10:30:00+05:00", true)] + [InlineData("10:30:00Z", true)] + public void TimeOnlyIsoValueReadsAsTheSameFractionInEitherDateSystem(string text, bool date1904) + { + using MemoryStream ms = WorkbookBuilder.Build( + $"""{text}""", date1904: date1904); + using XlsxWorkbook reader = Excel.FromXlsx(ms); + using XlsxWorkbook.Enumerator e = reader.FirstSheet.GetEnumerator(); + + Assert.True(e.MoveNext()); + Assert.Equal(CellType.Date, e.Current[0].Type); + Assert.Equal("0.4375", e.Current[0].GetString()); + Assert.True(e.Current[0].TryGetDateTime(reader.IsDate1904, out DateTime dt)); + Assert.Equal(TimeSpan.FromHours(10.5), dt.TimeOfDay); + } + + [Fact] + public void PrefixedSheetIsReadWhenAnXmlCommentContainsATag() + { + using MemoryStream ms = Package( + sheet: $"""5""", + sharedStrings: null); + using XlsxWorkbook reader = Excel.FromXlsx(ms); + using XlsxWorkbook.Enumerator e = reader.FirstSheet.GetEnumerator(); + + Assert.True(e.MoveNext()); + Assert.Equal("5", e.Current[0].GetString()); + } + + [Fact] + public void PrefixedSharedStringsAreReadWhenAnXmlCommentContainsATag() + { + using MemoryStream ms = Package( + sheet: $"""0""", + sharedStrings: $"""alpha"""); + using XlsxWorkbook reader = Excel.FromXlsx(ms); + using XlsxWorkbook.Enumerator e = reader.FirstSheet.GetEnumerator(); + + Assert.True(e.MoveNext()); + Assert.Equal("alpha", e.Current[0].GetString()); + } + + [Theory] + [InlineData("", "x:")] + [InlineData("?>", "x:")] + [InlineData("\">]>", "x:")] + [InlineData("", "")] + [InlineData("