diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml index 618abb59..67a3b9d0 100644 --- a/.github/workflows/benchmark-pages.yml +++ b/.github/workflows/benchmark-pages.yml @@ -43,6 +43,7 @@ jobs: - uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' cache: true cache-dependency-path: '**/*.csproj' diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index ecbbcad9..a03b6138 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -88,6 +88,7 @@ jobs: - uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' cache: true cache-dependency-path: '**/*.csproj' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e16ff7b..37f24ecf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,7 @@ jobs: uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' cache: true cache-dependency-path: '**/*.csproj' @@ -181,6 +182,7 @@ jobs: uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' cache: true cache-dependency-path: '**/*.csproj' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 14b6e992..946ebdc1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -37,6 +37,7 @@ jobs: - uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' # The snippets themselves are compiled by ci.yml, which builds the whole solution. - name: Check guide examples match docs/snippets diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index c17d96a5..79f5cd65 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -62,6 +62,7 @@ jobs: uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' cache: true cache-dependency-path: '**/*.csproj' @@ -155,6 +156,7 @@ jobs: uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' cache: true cache-dependency-path: '**/*.csproj' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9de1d5fc..86fb41c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ jobs: uses: actions/setup-dotnet@v6 with: global-json-file: global.json + dotnet-version: '10.0.x' cache: true cache-dependency-path: '**/*.csproj' diff --git a/.vscode/settings.json b/.vscode/settings.json index a6bf07ee..5b0ea647 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,7 +4,7 @@ "EXCELREADER_BUILD_BENCHMARKS": "ON", "EXCELREADER_BUILD_BENCHMARKS_COMPARE": "ON", "EXCELREADER_BUILD_TESTS": "ON", - "EXCELREADER_NATIVE_LIB": "${workspaceFolder}/src/ExcelReader.Native/bin/Release/net10.0/win-x64/native/ExcelReader.Native.dll" + "EXCELREADER_NATIVE_LIB": "${workspaceFolder}/src/ExcelReader.Native/bin/Release/net11.0/win-x64/native/ExcelReader.Native.dll" }, "cmake.configureArgs": [ "-DEXCELREADER_BUILD_BENCHMARKS=OFF", diff --git a/Directory.Build.props b/Directory.Build.props index af92fee5..821713bb 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -23,11 +23,11 @@ it used to arrive transitively through Roslyn.Diagnostics.Analyzers, which brought a whole Roslyn-internal ruleset along for those three rules. --> - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/docs/guide/reading.md b/docs/guide/reading.md index dc4fb0c5..b7d2dad3 100644 --- a/docs/guide/reading.md +++ b/docs/guide/reading.md @@ -128,7 +128,7 @@ A callback that writes to a database or sends messages must be idempotent on a C [`ExcelParseException.Row`](xref:ExcelReader.Core.Parser.ExcelParseException.Row) on `ParseAsync` counts the rows emitted so far. On `ForEachAsync` and `AggregateAsync` it is the row a sequential read reports, whether or not the read was partitioned. -See the [parallel sheet benchmarks](../performance/benchmarks.md#parallel-sheet-reading) and the [parallel CSV benchmarks](../performance/benchmarks.md#parallel-csv) for what this buys. On a real-data XLSX the sheet stage drops from 24 ms (prefetched, sequential) to about 17 ms, and on XLSB from 17 ms to about 13 ms; the floor is inflate, which runs on one thread. CSV has no such floor: its `AggregateAsync` at dop 4 is ~2.0x faster than the typed path while allocating ~630x less, with zero garbage collections. +See the [parallel sheet benchmarks](../performance/benchmarks.md#parallel-sheet-reading) and the [parallel CSV benchmarks](../performance/benchmarks.md#parallel-csv) for what this buys. On a real-data XLSX a typed aggregate drops from 44 ms on one thread to about 16 ms at dop 4, and on XLSB from 24 ms to about 13 ms; the floor is inflate, which runs on one thread. CSV has no such floor: its `AggregateAsync` at dop 4 is ~1.9x faster than the typed path while allocating ~670x less, with zero garbage collections. Properties need setters. A get-only property is skipped during binding and silently receives nothing. @@ -356,16 +356,16 @@ Measured across both read benchmarks (see [Real data reads](../performance/bench | Workload | Default | `PrefetchDecompression = true` | Gain | |---|---:|---:|---:| -| XLSX, real data | 37.2 ms | 23.7 ms | 36% | -| XLSM, real data | 38.0 ms | 23.1 ms | 39% | -| XLSB, real data | 24.7 ms | 16.6 ms | 33% | -| XLSX, string-heavy | 30.0 ms | 20.0 ms | 33% | -| XLSB, string-heavy | 33.6 ms | 19.5 ms | 42% | +| XLSX, real data | 38.4 ms | 25.0 ms | 35% | +| XLSM, real data | 37.8 ms | 24.5 ms | 35% | +| XLSB, real data | 18.9 ms | 10.4 ms | 45% | +| XLSX, string-heavy | 30.5 ms | 20.5 ms | 33% | +| XLSB, string-heavy | 26.4 ms | 16.8 ms | 36% | The gain tracks how much of a read is decompression rather than parsing, so it is smallest -on XLSB real data and string-heavy XLSX (about a third) and largest on string-heavy XLSB (42%). Allocations rise from the producer task and the -pooled decompression buffers — on the real-data corpus, roughly 7 KB to 23 KB for XLSX -and 7 KB to 15 KB for XLSB — and neither path triggers a garbage collection. +on string-heavy XLSX (about a third) and largest on XLSB real data (45%). Allocations rise from the producer task and the +pooled decompression buffers — on the real-data corpus, roughly 5 KB to 22 KB for XLSX +and 5 KB to 11 KB for XLSB — and neither path triggers a garbage collection. Do **not** enable it for concurrent server workloads: a caller already reading many files in parallel is CPU-saturated, and an extra background thread per read only doubles thread diff --git a/docs/guide/writing.md b/docs/guide/writing.md index e5cb6595..af5a57fe 100644 --- a/docs/guide/writing.md +++ b/docs/guide/writing.md @@ -368,10 +368,10 @@ Writing a 50,000-row workbook (`WriteBenchmark`): | Workload | Default | `PrefetchWrite = true` | Gain | |---|---:|---:|---:| -| XLSX | 8.191 ms | 5.351 ms | 35% | -| XLSB | 5.462 ms | 3.282 ms | 40% | +| XLSX | 8.213 ms | 5.577 ms | 32% | +| XLSB | 5.388 ms | 3.271 ms | 39% | -Allocations are unchanged (4.02 MB either way) — the background writer hands over buffers the row +Allocations are unchanged (4.02–4.03 MB either way) — the background writer hands over buffers the row writer already owns rather than copying them. The same caveat as the read side applies: **do not enable it for concurrent server workloads**. A @@ -433,13 +433,13 @@ Writing 1,000,000 rows of four columns (text, integer, date, number) to a `Memor | Write | XLSX | XLSB | |---|---:|---:| -| Serial | 165.08 ms | 113.61 ms | -| Serial, `PrefetchWrite` | 102.18 ms | 62.39 ms | -| Parallel, 1 batch in flight | 164.23 ms | 114.03 ms | -| Parallel, 2 | 84.48 ms | 58.53 ms | -| Parallel, 4 | 44.27 ms | 30.04 ms | -| Parallel, 8 | 27.54 ms | 19.48 ms | -| Parallel, 16 | 21.64 ms | 15.37 ms | +| Serial | 160.09 ms | 110.06 ms | +| Serial, `PrefetchWrite` | 99.24 ms | 61.54 ms | +| Parallel, 1 batch in flight | 174.04 ms | 107.89 ms | +| Parallel, 2 | 87.64 ms | 53.74 ms | +| Parallel, 4 | 45.61 ms | 28.72 ms | +| Parallel, 8 | 28.93 ms | 18.66 ms | +| Parallel, 16 | 22.07 ms | 15.11 ms | The file comes out 0.18% (XLSX) or 0.04% (XLSB) larger, because each batch is compressed on its own. As with `PrefetchWrite`, this is for single-file batch work: a server already writing many files at diff --git a/docs/performance/benchmarks.md b/docs/performance/benchmarks.md index 78e19b4f..71d089c2 100644 --- a/docs/performance/benchmarks.md +++ b/docs/performance/benchmarks.md @@ -5,9 +5,9 @@ Throwaway numbers go stale; these are regenerated from the benchmark suite in ## Benchmarks -Benchmarks were run with BenchmarkDotNet v0.16.0-preview.1 on Windows 10 (22H2), AMD Ryzen 7 5700X, .NET 10.0.13 (built with SDK 11.0.100-rc.2, targeting `net10.0`). Generated-data benchmarks use 50,000 rows, except the string-heavy reads, which use 65,536. Raw results: [GitHub Pages benchmark dashboard](https://gabrielmarquezmatte.github.io/ExcelReader/dev/bench/) — `tests/ExcelReader.Benchmarks/BenchmarkDotNet.Artifacts/` is `.gitignore`-excluded and never actually reaches GitHub, so a link to it 404s for every reader. +Benchmarks were run with BenchmarkDotNet v0.16.0-preview.2 on Windows 10 (22H2), AMD Ryzen 7 5700X, .NET 10.0.14 (built with SDK 10.0.403, targeting `net10.0`). Generated-data benchmarks use 50,000 rows, except the string-heavy reads, which use 65,536. Raw results: [GitHub Pages benchmark dashboard](https://gabrielmarquezmatte.github.io/ExcelReader/dev/bench/) — `tests/ExcelReader.Benchmarks/BenchmarkDotNet.Artifacts/` is `.gitignore`-excluded and never actually reaches GitHub, so a link to it 404s for every reader. -Every BenchmarkDotNet table is a `--job Short` run (1 launch, 3 warmup and 3 measured iterations) on an otherwise idle machine. Methods whose 99.9% confidence interval came out wider than 10% of the mean on any row were run again with `--job Medium` (2 launches, 15 iterations) and carry those figures instead: the parallel CSV tables (except Sep's sequential rows), the large-sheet tables, the parallel sheet read, the serial, `PrefetchWrite`, 2-batch XLSB and 16-batch XLSX parallel writes, the XLS and XLSX rows of the XLS writer comparison, the XLS typed record write, the XLSB `PrefetchWrite` write, the XLSX real-data matched-work read, the string-heavy XLSX matched-work read, the interned XLSB matched-work read and the Sylvan XLSX string-heavy read, the stored whole read, the inferred-schema Arrow conversion, the decoded native read-all and the batched native typed reads. The read and typed-parse rows of the XLSX and XLSB tables, the generated-workbook matched-work read, the `ref struct` parsing table, the ADO.NET bridge table and the encrypted-workbook table come from later `--job Short --launchCount 5` runs instead: Tiered PGO settles each process into a different code shape, which moved these rows by up to ±10% between single-launch runs. The cold-start rows set their own launch count, and the file-size and working-set figures under the parallel record writes, the managed deflate and CRC-32 figures, the Python/C++/Rust figures and the native-library comparison come from other harnesses that were not re-run. +Every BenchmarkDotNet table is one `--job Medium` run of the whole suite (2 launches, 10 warmup and 15 measured iterations) on an otherwise idle machine. Tiered PGO settles each process into a different code shape, so a row can still move by up to ±10% between runs; compare rows within a table, not across releases. The cold-start rows set their own launch count, and the file-size and working-set figures under the parallel record writes, the managed deflate and CRC-32 figures, the Python/C++/Rust figures and the native-library comparison come from other harnesses that were not re-run. **Benchmark methodology.** In the Excel-format tables below, the **"Cell-by-cell read"** rows (including the per-format rows in "Real data reads" / "String-heavy reads") read ExcelReader's `cell.Value` — a zero-copy `ReadOnlySpan`, no decode or allocation — against each competitor's own idiomatic read call. For Sylvan.Data.Excel, that's the ADO.NET-style `GetString(i)`, which is forced to materialize a UTF-16 `string`; its API has no zero-copy accessor, so it cannot avoid that cost. These rows are therefore not matched work: part of the reported gap is "we parse faster" and part is "we skipped an allocation you were never offered a way to skip." Each affected benchmark class also has a `*_Materialized` sibling (calling `cell.GetString()`, the same UTF-16 materialization Sylvan pays) so the matched-work number is measurable — see `tests/ExcelReader.Benchmarks/Shared/BenchmarkAccumulators.cs`. **Those results are published under [Matched-work reads](#matched-work-reads-_materialized), and reading them is the honest way to judge the comparison:** treat the Excel cell-by-cell ratios in the tables below as an upper bound, not a like-for-like number. The CSV cell-by-cell rows are matched work: Sep and Sylvan.Data.Csv both expose span accessors (`row[i].Span`, `GetFieldSpan(i)`), and the benchmarks use them. The **"Typed row parsing"** / **"Typed record writing"** rows are unaffected — both sides already materialize real objects/strings there, so those comparisons are matched work as published. @@ -17,55 +17,55 @@ Compares ExcelReader against established XLSX libraries on the same generated wo | Scenario | ExcelReader | Sylvan | ExcelDataReader | OfficeIMO | SpreadCheetah | |---|---:|---:|---:|---:|---:| -| Cell-by-cell read | 7.098 ms, 3.44 KB | 37.056 ms, 1.89 MB | 87.075 ms, 32.88 MB | 13.892 ms, 3.51 MB | - | -| Cell-by-cell read async | 7.428 ms, 3.51 KB | - | - | - | - | -| Typed row parsing | 10.913 ms, 3.87 MB | 60.050 ms, 10.48 MB | - | 11.790 ms, 2.36 MB | - | -| Typed row parsing async | 10.223 ms, 3.87 MB | 58.884 ms, 10.48 MB | - | - | - | -| Typed row parsing, shared strings | 7.737 ms, 2.29 MB | - | - | - | - | -| Typed row parsing, `InternStrings` | 10.633 ms, 2.30 MB | - | - | - | - | -| Workbook writing | 8.191 ms, 4.02 MB | - | - | 18.162 ms, 4.04 MB | 14.828 ms, 15.84 MB | -| Workbook writing, shared strings | 7.981 ms, 4.03 MB | - | - | - | - | +| Cell-by-cell read | 6.363 ms, 3.48 KB | 37.532 ms, 1.89 MB | 90.746 ms, 32.88 MB | 12.554 ms, 3.51 MB | - | +| Cell-by-cell read async | 6.613 ms, 3.55 KB | - | - | - | - | +| Typed row parsing | 8.875 ms, 3.87 MB | 57.041 ms, 10.48 MB | - | 10.327 ms, 2.36 MB | - | +| Typed row parsing async | 10.120 ms, 3.87 MB | 60.316 ms, 10.48 MB | - | - | - | +| Typed row parsing, shared strings | 7.629 ms, 2.29 MB | - | - | - | - | +| Typed row parsing, `InternStrings` | 9.103 ms, 2.34 MB | - | - | - | - | +| Workbook writing | 8.213 ms, 4.02 MB | - | - | 18.105 ms, 4.04 MB | 15.310 ms, 15.84 MB | +| Workbook writing, shared strings | 7.965 ms, 4.03 MB | - | - | - | - | -ExcelReader is ~5.2x faster than Sylvan for raw XLSX reads, allocating ~564x less, ~12.3x faster than ExcelDataReader, and ~2.0x faster than OfficeIMO. For typed parsing it is ~5.5x faster than Sylvan (~5.8x async against async) and ~1.1x faster than OfficeIMO, which allocates less (2.36 MB against 3.87 MB) because it always interns strings. `ExcelReaderOptions.InternStrings` is off by default; turned on, ExcelReader allocates 2.30 MB at 10.633 ms, ~10% faster than OfficeIMO in the same run. For XLSX writing, ExcelReader is ~1.8x faster than SpreadCheetah while allocating ~3.9x less memory, and ~2.2x faster than OfficeIMO. The OfficeIMO rows are OfficeIMO.Excel 3.5.1, and the ExcelDataReader rows ExcelDataReader 3.9.0. +ExcelReader is ~5.9x faster than Sylvan for raw XLSX reads, allocating ~558x less, ~14.3x faster than ExcelDataReader, and ~2.0x faster than OfficeIMO. For typed parsing it is ~6.4x faster than Sylvan (~6.0x async against async) and ~1.2x faster than OfficeIMO, which allocates less (2.36 MB against 3.87 MB) because it always interns strings. `ExcelReaderOptions.InternStrings` is off by default; turned on, ExcelReader allocates 2.34 MB at 9.103 ms, ~12% faster than OfficeIMO in the same run. For XLSX writing, ExcelReader is ~1.9x faster than SpreadCheetah while allocating ~3.9x less memory, and ~2.2x faster than OfficeIMO. The OfficeIMO rows are OfficeIMO.Excel 3.5.3, and the ExcelDataReader rows ExcelDataReader 3.9.0. OfficeIMO.Excel's typed parsing is hand-mapped from its `OpenDataReader`, because `RowsAs` needs an `r` attribute on every row and cell, which the spec makes optional and ExcelReader's writer omits. -Reading a shared-strings XLSX workbook with typed parsing is ~29% faster than the inline-string sheet above (7.737 ms vs. 10.913 ms) and allocates ~41% less (2.29 MB vs. 3.87 MB) — each distinct string decodes once into the shared-string cache instead of once per cell occurrence. +Reading a shared-strings XLSX workbook with typed parsing is ~14% faster than the inline-string sheet above (7.629 ms vs. 8.875 ms) and allocates ~41% less (2.29 MB vs. 3.87 MB) — each distinct string decodes once into the shared-string cache instead of once per cell occurrence. ### XLSB (BIFF12) | Scenario | ExcelReader | ExcelDataReader | OfficeIMO | |---|---:|---:|---:| -| Cell-by-cell read | 4.942 ms, 3.67 KB | 28.106 ms, 18.06 MB | 9.841 ms, 5.12 MB | -| Cell-by-cell read async | 5.071 ms, 3.74 KB | - | - | -| Typed row parsing | 7.375 ms, 3.87 MB | - | - | -| Typed row parsing async | 7.276 ms, 3.87 MB | - | - | -| Workbook writing | 5.462 ms, 4.02 MB | - | - | -| Workbook writing, shared strings | 5.521 ms, 4.03 MB | - | - | +| Cell-by-cell read | 4.427 ms, 3.71 KB | 28.009 ms, 18.06 MB | 10.117 ms, 5.12 MB | +| Cell-by-cell read async | 4.432 ms, 3.78 KB | - | - | +| Typed row parsing | 7.070 ms, 3.87 MB | - | - | +| Typed row parsing async | 7.139 ms, 3.87 MB | - | - | +| Workbook writing | 5.388 ms, 4.02 MB | - | - | +| Workbook writing, shared strings | 5.377 ms, 4.03 MB | - | - | -XLSB is the fastest generated Excel format in these results: raw reads are ~1.4x faster than their XLSX counterparts, and typed parsing and writing ~1.5x. Against OfficeIMO's XLSB reader, ExcelReader is ~2.0x faster while allocating ~1,430x less, and against ExcelDataReader's ~5.7x faster. The XLSB writer is also ~2.7x faster than SpreadCheetah's XLSX writer on this benchmark while allocating ~75% less memory. +XLSB is the fastest generated Excel format in these results: raw reads are ~1.4x faster than their XLSX counterparts, typed parsing ~1.3x and writing ~1.5x. Against OfficeIMO's XLSB reader, ExcelReader is ~2.3x faster while allocating ~1,410x less, and against ExcelDataReader's ~6.3x faster. The XLSB writer is also ~2.8x faster than SpreadCheetah's XLSX writer on this benchmark while allocating ~75% less memory. ### XLS (BIFF8) | Scenario | ExcelReader | Sylvan | ExcelDataReader | OfficeIMO | |---|---:|---:|---:|---:| -| Cell-by-cell read | 3.529 ms, 2.84 KB | 5.199 ms, 1,717.73 KB | 38.718 ms, 43,522.77 KB | 7.629 ms, 10,499.44 KB | -| Cell-by-cell read async | 3.279 ms, 2.91 KB | - | - | - | -| Workbook writing | 4.736 ms, 16.31 MB | - | - | - | +| Cell-by-cell read | 2.956 ms, 3.14 KB | 5.369 ms, 1,717.73 KB | 38.608 ms, 43,522.77 KB | 7.941 ms, 10,499.44 KB | +| Cell-by-cell read async | 3.498 ms, 3.21 KB | - | - | - | +| Workbook writing | 4.557 ms, 16.32 MB | - | - | - | -ExcelReader is ~1.5x faster than Sylvan for generated XLS reads while allocating ~605x less memory, ~2.2x faster than OfficeIMO, and ~11x faster than ExcelDataReader. The XLS writer is ~1.7x faster than the XLSX writer in this benchmark (`XlsWriteBenchmark`, 4.736 ms vs 7.941 ms in the same run). Most of its 16.31 MB is the benchmark's pre-sized 16 MB destination `MemoryStream`; the typed-record table below measures it at 4.31 MB, close to the other formats. +ExcelReader is ~1.8x faster than Sylvan for generated XLS reads while allocating ~547x less memory, ~2.7x faster than OfficeIMO, and ~13x faster than ExcelDataReader. The XLS writer is ~1.8x faster than the XLSX writer in this benchmark (`XlsWriteBenchmark`, 4.557 ms vs 8.243 ms in the same run). Most of its 16.32 MB is the benchmark's pre-sized 16 MB destination `MemoryStream`; the typed-record table below measures it at 4.32 MB, close to the other formats. ### CSV | Scenario | ExcelReader | Sep | Sylvan.Data.Csv | |---|---:|---:|---:| -| Cell-by-cell read | 2.986 ms, 888 B | 8.005 ms, 3.93 KB | 4.034 ms, 37.78 KB | -| Cell-by-cell read async | 2.907 ms, 960 B | - | - | -| Typed row parsing | 4.618 ms, 3.86 MB | 8.438 ms, 3.87 MB | 11.703 ms, 10.95 MB | -| Typed row parsing async | 4.876 ms, 3.86 MB | - | - | -| Row writing | 4.634 ms, 4.00 MB | 6.861 ms, 4.01 MB | 6.925 ms, 4.04 MB | +| Cell-by-cell read | 3.080 ms, 896 B | 7.814 ms, 3.93 KB | 4.109 ms, 37.78 KB | +| Cell-by-cell read async | 3.359 ms, 968 B | - | - | +| Typed row parsing | 4.474 ms, 3.86 MB | 8.588 ms, 3.87 MB | 12.068 ms, 10.95 MB | +| Typed row parsing async | 4.840 ms, 3.86 MB | - | - | +| Row writing | 4.660 ms, 4.00 MB | 6.979 ms, 4.01 MB | 7.149 ms, 4.04 MB | -For raw CSV reads, all three libraries read text through spans. ExcelReader is ~2.7x faster than Sep while allocating ~4.5x less, and ~1.4x faster than Sylvan.Data.Csv while allocating ~44x less. For typed CSV parsing (the more common case — building actual records), ExcelReader is ~1.8x faster than Sep and ~2.5x faster than Sylvan.Data.Csv, with the lowest allocation of the group. For CSV writing, ExcelReader is ~1.5x faster than both Sep and Sylvan.Data.Csv, which are tied; the ~4 MB shown across all three is primarily the benchmark's pre-sized destination `MemoryStream`, not per-row writer state. +For raw CSV reads, all three libraries read text through spans. ExcelReader is ~2.5x faster than Sep while allocating ~4.5x less, and ~1.3x faster than Sylvan.Data.Csv while allocating ~43x less. For typed CSV parsing (the more common case — building actual records), ExcelReader is ~1.9x faster than Sep and ~2.7x faster than Sylvan.Data.Csv, with the lowest allocation of the group. For CSV writing, ExcelReader is ~1.5x faster than both Sep and Sylvan.Data.Csv, which are tied; the ~4 MB shown across all three is primarily the benchmark's pre-sized destination `MemoryStream`, not per-row writer state. ### Parallel CSV @@ -73,34 +73,34 @@ For raw CSV reads, all three libraries read text through spans. ExcelReader is ~ | Dop | Conversion-heavy, typed | Conversion-heavy, struct + `InternStrings` | Narrow ints, typed | Conversion-heavy, `ref struct` aggregate | Narrow ints, `ref struct` aggregate | |---:|---:|---:|---:|---:|---:| -| 1 | 612.4 ms, 671.42 MB | 561.2 ms, 1.64 MB | 512.1 ms, 245.76 MB | 453.5 ms, 1.21 MB | 410.6 ms, 1.20 MB | -| 2 | 472.5 ms, 680.04 MB | 447.5 ms, 10.86 MB | 399.0 ms, 254.48 MB | 263.3 ms, 0.91 MB | 245.3 ms, 0.92 MB | -| 4 | 268.9 ms, 680.10 MB | 254.9 ms, 11.33 MB | 224.3 ms, 254.56 MB | 136.1 ms, 1.07 MB | 130.5 ms, 1.09 MB | -| 8 | 219.9 ms, 680.10 MB | 197.2 ms, 12.21 MB | 193.7 ms, 254.52 MB | 92.2 ms, 1.27 MB | 93.3 ms, 1.26 MB | -| 16 | 419.6 ms, 680.12 MB | 171.3 ms, 14.17 MB | 255.5 ms, 255.02 MB | 68.1 ms, 1.38 MB | 70.5 ms, 1.34 MB | +| 1 | 574.1 ms, 671.41 MB | 547.7 ms, 1.68 MB | 464.7 ms, 245.75 MB | 453.4 ms, 1.21 MB | 432.2 ms, 1.21 MB | +| 2 | 452.2 ms, 679.93 MB | 452.6 ms, 10.86 MB | 405.1 ms, 254.36 MB | 253.6 ms, 0.85 MB | 247.7 ms, 0.86 MB | +| 4 | 257.8 ms, 680.00 MB | 248.7 ms, 11.43 MB | 230.5 ms, 254.43 MB | 137.2 ms, 1.02 MB | 130.1 ms, 1.02 MB | +| 8 | 208.1 ms, 679.99 MB | 189.7 ms, 12.28 MB | 192.1 ms, 254.41 MB | 90.7 ms, 1.20 MB | 90.1 ms, 1.19 MB | +| 16 | 486.8 ms, 680.03 MB | 167.5 ms, 14.31 MB | 268.3 ms, 254.91 MB | 67.2 ms, 1.32 MB | 70.1 ms, 1.28 MB | -The typed path scales ~2.3x by dop 4 on both corpora (612.4 to 268.9 ms conversion-heavy, 512.1 to 224.3 ms narrow), then flattens out: narrow ints reach 193.7 ms at dop 8 and slow to 255.5 ms at dop 16, and the conversion-heavy corpus gets *slower* past dop 8 (419.6 ms at dop 16 against 219.9 ms at dop 8, standard deviation 105.4 ms). Its ~680 MB is one materialized row object and two strings per record, and at dop 16 that costs 42,000 Gen0 plus 41,000 Gen1 collections. +The typed path scales ~2.2x by dop 4 on the conversion-heavy corpus and ~2.0x on the narrow one (574.1 to 257.8 ms, 464.7 to 230.5 ms), then flattens out: narrow ints reach 192.1 ms at dop 8 and slow to 268.3 ms at dop 16, and the conversion-heavy corpus gets *slower* past dop 8 (486.8 ms at dop 16 against 208.1 ms at dop 8, standard deviation 24.1 ms). Its ~680 MB is one materialized row object and two strings per record, and at dop 16 that costs 42,000 Gen0 plus 41,000 Gen1 collections. -The struct + `InternStrings` column parses the same rows into a `struct` with `ExcelReaderOptions.InternStrings` on, so there is no row object and each repeated region or country name is one cached string. It never slows down: 197.2 ms at dop 8, 171.3 ms at dop 16, 2.4x faster than the class at dop 16 while allocating ~48x less (14.17 MB, under one Gen0 collection per run). What remains is per-worker caches and the ordered merge's buffers. The interning win depends on the data: the corpus has 4 regions and 6 countries, so every string is a cache hit. Text columns with mostly distinct values (names, ids, free text) allocate a string per cell either way, and on those the struct alone removes only the row object. +The struct + `InternStrings` column parses the same rows into a `struct` with `ExcelReaderOptions.InternStrings` on, so there is no row object and each repeated region or country name is one cached string. It never slows down: 189.7 ms at dop 8, 167.5 ms at dop 16, 2.9x faster than the class at dop 16 while allocating ~48x less (14.31 MB, under one Gen0 collection per run). What remains is per-worker caches and the ordered merge's buffers. The interning win depends on the data: the corpus has 4 regions and 6 countries, so every string is a cache hit. Text columns with mostly distinct values (names, ids, free text) allocate a string per cell either way, and on those the struct alone removes only the row object. -The aggregate columns fold each record into a per-partition accumulator through a `ref struct` model, so text columns stay as `ReadOnlySpan` and no row object is ever materialized. They scale better than the typed path — ~4.9x (conversion-heavy) and ~4.4x (narrow) at dop 8, ~6.7x and ~5.8x at dop 16 — because they are not fighting the allocator: at dop 4 the conversion-heavy aggregate is ~2.0x faster than typed while allocating ~630x less (1.07 MB against 680.10 MB), with **zero** garbage collections at every degree. The remaining allocation is per-partition bookkeeping, not per-row. +The aggregate columns fold each record into a per-partition accumulator through a `ref struct` model, so text columns stay as `ReadOnlySpan` and no row object is ever materialized. They scale better than the typed path — ~5.0x (conversion-heavy) and ~4.8x (narrow) at dop 8, ~6.7x and ~6.2x at dop 16 — because they are not fighting the allocator: at dop 4 the conversion-heavy aggregate is ~1.9x faster than typed while allocating ~670x less (1.02 MB against 680.00 MB), with **zero** garbage collections at every degree. The remaining allocation is per-partition bookkeeping, not per-row. -The typed path's slowdown past dop 8 is the garbage collector. Every degree allocates the same ~680 MB and runs 42 gen0 collections, but at dop 16 each of them also collects gen1 (41 gen1 collections against 26 at dop 8), and the pauses add up to 100-200 ms against 39 ms. In a console harness on the 4.3 million row corpus, with the default workstation GC, dop 16 took 365 ms; with `DOTNET_gcServer=1` it took 182 ms, faster than dop 8. Not allocating per row, as the struct + `InternStrings` and aggregate columns do, avoids it without changing the GC mode. +The typed path's slowdown past dop 8 is the garbage collector. Every degree allocates the same ~680 MB and runs 42 gen0 collections, but at dop 16 each of them also collects gen1 (41 gen1 collections against 24 at dop 8), and the pauses add up to 100-200 ms against 39 ms. In a console harness on the 4.3 million row corpus, with the default workstation GC, dop 16 took 365 ms; with `DOTNET_gcServer=1` it took 182 ms, faster than dop 8. Not allocating per row, as the struct + `InternStrings` and aggregate columns do, avoids it without changing the GC mode. -Type conversion is the floor under all of them. A single-threaded pass that allocates nothing still costs 453.5 ms, so parsing bytes into `DateTime`/`decimal`/`int` — not allocation and not I/O — is what the parallelism is actually buying down. +Type conversion is the floor under all of them. A single-threaded pass that allocates nothing still costs 453.4 ms, so parsing bytes into `DateTime`/`decimal`/`int` — not allocation and not I/O — is what the parallelism is actually buying down. Against [Sep](https://github.com/nietras/Sep)'s own `ParallelEnumerate` (all cores; 8,000,000 narrow rows, 3,000,000 conversion-heavy rows): | Corpus, row type | ExcelReader sequential | ExcelReader parallel | Sep sequential | Sep parallel | |---|---:|---:|---:|---:| -| Narrow ints, class | 474.9 ms, 244.15 MB | 269.1 ms, 255.02 MB | 507.5 ms, 244.15 MB | 433.0 ms, 245.46 MB | -| Narrow ints, struct | 392.0 ms, 13.93 KB | 162.0 ms, 12.15 MB | 476.2 ms, 9.34 KB | 251.7 ms, 2.81 MB | -| Conversion-heavy, class | 407.6 ms, 467.31 MB | 353.3 ms, 474.68 MB | 835.3 ms, 467.31 MB | 596.5 ms, 468.41 MB | -| Conversion-heavy, struct + pooled strings | 377.3 ms, 14.55 KB | 119.6 ms, 11.28 MB | 857.6 ms, 10.77 KB | 392.7 ms, 9.33 MB | +| Narrow ints, class | 451.0 ms, 244.14 MB | 271.5 ms, 255.06 MB | 502.0 ms, 244.15 MB | 446.5 ms, 247.41 MB | +| Narrow ints, struct | 444.6 ms, 38.24 KB | 153.0 ms, 11.90 MB | 480.0 ms, 9.34 KB | 257.3 ms, 4.22 MB | +| Conversion-heavy, class | 375.6 ms, 467.30 MB | 320.6 ms, 474.61 MB | 830.7 ms, 467.31 MB | 602.3 ms, 469.10 MB | +| Conversion-heavy, struct + pooled strings | 389.5 ms, 75.22 KB | 114.7 ms, 11.41 MB | 918.5 ms, 10.77 KB | 380.5 ms, 5.01 MB | -The struct rows give both libraries the same help: the row is a `struct`, and repeated text is pooled, through `ExcelReaderOptions.InternStrings` on ExcelReader's side and `SepToString.PoolPerCol()` (sequential) or `PoolPerColThreadSafe()` (parallel) on Sep's. Every row except Sep's sequential ones is from a Medium job, since the Short run spread too much; Sep's conversion-heavy parallel struct row still has a 35.1 ms standard deviation, and ExcelReader's conversion-heavy parallel class row 29.4 ms. +The struct rows give both libraries the same help: the row is a `struct`, and repeated text is pooled, through `ExcelReaderOptions.InternStrings` on ExcelReader's side and `SepToString.PoolPerCol()` (sequential) or `PoolPerColThreadSafe()` (parallel) on Sep's. The parallel rows spread the most: ExcelReader's narrow and conversion-heavy parallel class rows have a 33.3 ms and 28.0 ms standard deviation, and Sep's conversion-heavy parallel struct row 27.6 ms. -Sequentially, ExcelReader's typed parser is ~1.1x faster than Sep on narrow ints (474.9 ms vs. 507.5 ms), and ~2.0x faster on the conversion-heavy corpus, where decimal parsing dominates. In parallel ExcelReader is ~1.6x (narrow) and ~1.7x (conversion-heavy) faster than Sep's, while yielding rows in file order. With struct rows and pooled strings both libraries speed up and stop collecting, and ExcelReader stays ahead: ~1.2x and ~2.3x sequentially, ~1.6x and ~3.3x in parallel. The class rows gain less from parallelism (353.3 ms against 407.6 ms sequential on the conversion-heavy corpus) because they are bound by the allocator, like the typed dop 16 row above: 28 Gen1 collections per run; the struct rows run 3.2x faster in parallel on the same corpus. As in the table above, pooling pays off here because the conversion-heavy corpus repeats 4 regions and 6 countries. +Sequentially, ExcelReader's typed parser is ~1.1x faster than Sep on narrow ints (451.0 ms vs. 502.0 ms), and ~2.2x faster on the conversion-heavy corpus, where decimal parsing dominates. In parallel ExcelReader is ~1.6x (narrow) and ~1.9x (conversion-heavy) faster than Sep's, while yielding rows in file order. With struct rows and pooled strings both libraries stop collecting, and ExcelReader stays ahead: ~1.1x and ~2.4x sequentially, ~1.7x and ~3.3x in parallel. The class rows gain less from parallelism (320.6 ms against 375.6 ms sequential on the conversion-heavy corpus) because they are bound by the allocator, like the typed dop 16 row above: ~28 Gen1 collections per run; the struct rows run 3.4x faster in parallel on the same corpus. As in the table above, pooling pays off here because the conversion-heavy corpus repeats 4 regions and 6 countries. ### Parallel sheet reading @@ -108,12 +108,12 @@ Sequentially, ExcelReader's typed parser is ~1.1x faster than Sep on narrow ints | Dop | XLSX | XLSB | |---:|---:|---:| -| 1 | 46.07 ms, 8.02 MB | 29.86 ms, 8.02 MB | -| 2 | 29.36 ms, 8.13 MB | 22.49 ms, 8.07 MB | -| 4 | 16.69 ms, 8.12 MB | 13.28 ms, 8.07 MB | -| 8 | 16.79 ms, 8.17 MB | 13.18 ms, 8.09 MB | +| 1 | 44.39 ms, 8.02 MB | 24.34 ms, 8.02 MB | +| 2 | 28.55 ms, 8.14 MB | 15.01 ms, 8.07 MB | +| 4 | 16.13 ms, 8.11 MB | 12.91 ms, 8.07 MB | +| 8 | 16.25 ms, 8.17 MB | 12.96 ms, 8.09 MB | -At dop 4 XLSX reads ~2.8x faster than sequentially and XLSB ~2.2x. Dop 2 is a single parse worker running alongside the inflating thread. Past dop 4 nothing improves: the one inflating thread is the floor (~14 ms for XLSX, ~9 ms for XLSB, see [Inflate](#inflate), plus the XLSB record walk that finds row boundaries). Allocation barely moves with dop (8.02 to 8.17 MB): it is the typed model's strings. +At dop 4 XLSX reads ~2.8x faster than sequentially and XLSB ~1.9x. Dop 2 is a single parse worker running alongside the inflating thread. Past dop 4 nothing improves: the one inflating thread is the floor (~14 ms for XLSX, ~9 ms for XLSB, see [Inflate](#inflate), plus the XLSB record walk that finds row boundaries). Allocation barely moves with dop (8.02 to 8.17 MB): it is the typed model's strings. ### Large sheets @@ -121,31 +121,31 @@ At dop 4 XLSX reads ~2.8x faster than sequentially and XLSB ~2.2x. Dop 2 is a si | XLSX, Dop | Aggregate | Typed parse | |---:|---:|---:| -| 1 | 223.16 ms, 77.25 MB | 249.45 ms, 77.25 MB | -| 2 | 163.11 ms, 77.62 MB | 190.50 ms, 106.81 MB | -| 4 | 59.25 ms, 77.57 MB | 73.08 ms, 106.75 MB | -| 8 | 45.08 ms, 77.64 MB | 55.79 ms, 106.78 MB | -| 16 | 44.48 ms, 77.95 MB | 55.15 ms, 107.11 MB | +| 1 | 179.48 ms, 77.25 MB | 205.45 ms, 77.25 MB | +| 2 | 134.76 ms, 77.68 MB | 145.41 ms, 106.87 MB | +| 4 | 48.73 ms, 77.58 MB | 60.19 ms, 106.78 MB | +| 8 | 39.57 ms, 77.66 MB | 48.65 ms, 106.84 MB | +| 16 | 39.85 ms, 78.00 MB | 50.17 ms, 107.17 MB | | XLSB, Dop | Aggregate | Typed parse | |---:|---:|---:| -| 1 | 157.36 ms, 77.25 MB | 170.55 ms, 77.25 MB | -| 2 | 108.82 ms, 77.53 MB | 118.56 ms, 99.69 MB | -| 4 | 47.71 ms, 77.48 MB | 56.25 ms, 99.62 MB | -| 8 | 49.58 ms, 77.61 MB | 54.14 ms, 99.76 MB | -| 16 | 49.46 ms, 77.87 MB | 53.82 ms, 100.03 MB | +| 1 | 130.43 ms, 77.25 MB | 155.74 ms, 77.25 MB | +| 2 | 95.28 ms, 77.57 MB | 104.01 ms, 99.73 MB | +| 4 | 45.25 ms, 77.48 MB | 51.91 ms, 99.62 MB | +| 8 | 45.65 ms, 77.62 MB | 52.67 ms, 99.76 MB | +| 16 | 45.75 ms, 77.88 MB | 53.06 ms, 100.02 MB | | CSV, Dop | Aggregate | Typed parse | Typed parse, struct + `InternStrings` | |---:|---:|---:|---:| -| 1 | 418.65 ms, 1.99 KB | 509.94 ms, 623.07 MB | 506.93 ms, 14.05 KB | -| 2 | 204.59 ms, 11.58 KB | 325.09 ms, 626.87 MB | 323.26 ms, 4.46 MB | -| 4 | 115.50 ms, 10.94 KB | 200.87 ms, 626.68 MB | 178.60 ms, 4.70 MB | -| 8 | 84.80 ms, 18.88 KB | 165.18 ms, 626.40 MB | 139.31 ms, 5.33 MB | -| 16 | 61.33 ms, 35.14 KB | 216.44 ms, 626.55 MB | 122.48 ms, 7.40 MB | +| 1 | 500.39 ms, 1.99 KB | 497.73 ms, 623.07 MB | 508.79 ms, 74.70 KB | +| 2 | 206.19 ms, 7.05 KB | 326.13 ms, 626.88 MB | 321.23 ms, 4.50 MB | +| 4 | 106.38 ms, 10.94 KB | 180.35 ms, 626.72 MB | 169.94 ms, 4.80 MB | +| 8 | 76.57 ms, 25.73 KB | 154.98 ms, 626.40 MB | 127.37 ms, 5.48 MB | +| 16 | 58.07 ms, 35.14 KB | 201.60 ms, 626.54 MB | 120.13 ms, 7.65 MB | The XLSX and XLSB aggregates and typed parses each materialize a `Record` and its name string per row, which is where their ~77-107 MB comes from; the CSV aggregate binds a `ref struct` and allocates kilobytes, while the CSV typed parse allocates ~626 MB at every degree, one row object and two strings per record. The struct + `InternStrings` parse drops both (the corpus repeats 4 regions and 6 countries, see [Parallel CSV](#parallel-csv)) and keeps a few MB of per-worker caches and merge buffers. -XLSX and XLSB reach ~3.0-3.8x at dop 4. The XLSB aggregate stops there (3.3x), as in the real-data table above, while the XLSX aggregate improves to 5.0x at dop 8 and then stops. The CSV aggregate keeps scaling because nothing but the parser runs on the workers: 4.9x at dop 8 and 6.8x at dop 16. The typed CSV parse reaches 3.1x at dop 8 and then loses 31% at dop 16 to the collector, while the struct + `InternStrings` parse keeps going to 4.1x at dop 16, 1.8x faster than the class. The aggregate is still the fastest, ~2.0x ahead of the struct parse at dop 16, because it never builds the row at all. +XLSX and XLSB reach ~2.9-3.7x at dop 4. The XLSB aggregate stops there (2.9x), as in the real-data table above, while the XLSX aggregate improves to 4.5x at dop 8 and then stops. The CSV aggregate keeps scaling because nothing but the parser runs on the workers: 206.19 ms at dop 2, 76.57 ms at dop 8 and 58.07 ms at dop 16. Its dop 1 row (500.39 ms) is no faster than the typed parse at dop 1 in this run, so ratios against it would overstate the scaling. The typed CSV parse reaches 3.2x at dop 8 and then loses 30% at dop 16 to the collector, while the struct + `InternStrings` parse keeps going to 4.2x at dop 16, 1.7x faster than the class. The aggregate is still the fastest, ~2.1x ahead of the struct parse at dop 16, because it never builds the row at all. ### Real data reads @@ -153,13 +153,13 @@ This benchmark reads a real workbook exported in multiple formats. | Format | ExcelReader | ExcelReader, prefetch | Sylvan | |---|---:|---:|---:| -| XLSX | 37.237 ms, 7.20 KB | 23.663 ms, 23.34 KB | 195.758 ms, 644.27 KB | -| XLSM | 37.999 ms, 7.20 KB | 23.061 ms, 23.35 KB | 199.923 ms, 644.35 KB | -| XLSB | 24.746 ms, 6.72 KB | 16.642 ms, 15.09 KB | 29.421 ms, 338.55 KB | -| XLS | 11.196 ms, 12.02 KB | n/a | 18.221 ms, 185.90 KB | -| CSV | 4.949 ms, 888 B | n/a | 4.502 ms, 40.26 KB | +| XLSX | 38.378 ms, 5.46 KB | 24.991 ms, 21.60 KB | 194.303 ms, 644.27 KB | +| XLSM | 37.822 ms, 5.46 KB | 24.501 ms, 21.52 KB | 192.195 ms, 644.33 KB | +| XLSB | 18.939 ms, 4.98 KB | 10.395 ms, 11.35 KB | 30.055 ms, 338.55 KB | +| XLS | 9.383 ms, 10.55 KB | n/a | 18.110 ms, 185.90 KB | +| CSV | 4.958 ms, 896 B | n/a | 4.595 ms, 40.26 KB | -On this real-data workload, ExcelReader is ~5.3x faster than Sylvan for XLSX and XLSM, ~1.2x faster for XLSB, and ~1.6x faster for XLS — allocating ~90x less for XLSX and XLSM, ~50x less for XLSB and ~15x less for XLS. On CSV, where both sides read text through spans, Sylvan.Data.Csv is ~10% faster, while ExcelReader allocates ~46x less. The prefetch column is the opt-in [`PrefetchDecompression`](../guide/reading.md#prefetch-decompression-xlsxxlsb) option; XLS and CSV are uncompressed, so it does not apply to them. +On this real-data workload, ExcelReader is ~5.1x faster than Sylvan for XLSX and XLSM, ~1.6x faster for XLSB, and ~1.9x faster for XLS — allocating ~118x less for XLSX and XLSM, ~68x less for XLSB and ~18x less for XLS. On CSV, where both sides read text through spans, Sylvan.Data.Csv is ~7% faster, while ExcelReader allocates ~46x less. The prefetch column is the opt-in [`PrefetchDecompression`](../guide/reading.md#prefetch-decompression-xlsxxlsb) option; XLS and CSV are uncompressed, so it does not apply to them. ### In-memory real-data reads @@ -167,18 +167,18 @@ The real-data benchmark also measures the in-memory path for workbook content lo | Method | Mean | StdDev | Allocated | |---|---:|---:|---:| -| Xlsx_ExcelReader_Memory | 37.534 ms | 0.007 ms | 6.71 KB | -| Xlsx_ExcelReader_Memory_Prefetch | 23.609 ms | 0.043 ms | 22.83 KB | -| Xlsm_ExcelReader_Memory | 37.225 ms | 0.045 ms | 6.71 KB | -| Xlsm_ExcelReader_Memory_Prefetch | 23.467 ms | 0.052 ms | 22.78 KB | -| Xlsb_ExcelReader_Memory | 24.418 ms | 0.012 ms | 8.31 KB | -| Xlsb_ExcelReader_Memory_Prefetch | 15.382 ms | 0.047 ms | 16.58 KB | -| Xls_ExcelReader_Memory | 11.031 ms | 0.051 ms | 11.87 KB | -| Csv_ExcelReader_Memory | 4.744 ms | 0.004 ms | 616 B | +| Xlsx_ExcelReader_Memory | 38.792 ms | 0.197 ms | 5.09 KB | +| Xlsx_ExcelReader_Memory_Prefetch | 24.980 ms | 0.390 ms | 21.17 KB | +| Xlsm_ExcelReader_Memory | 38.033 ms | 0.208 ms | 5.09 KB | +| Xlsm_ExcelReader_Memory_Prefetch | 24.836 ms | 0.080 ms | 21.17 KB | +| Xlsb_ExcelReader_Memory | 18.590 ms | 0.083 ms | 4.70 KB | +| Xlsb_ExcelReader_Memory_Prefetch | 10.433 ms | 0.049 ms | 9.20 KB | +| Xls_ExcelReader_Memory | 9.080 ms | 0.098 ms | 10.39 KB | +| Csv_ExcelReader_Memory | 4.761 ms | 0.004 ms | 616 B | -`Csv_ExcelReader_Memory` is both faster (4.744 ms vs. 4.949 ms for `Csv_ExcelReader`) and allocates less (616 B vs. 888 B) than its stream twin. +`Csv_ExcelReader_Memory` is both faster (4.761 ms vs. 4.958 ms for `Csv_ExcelReader`) and allocates less (616 B vs. 896 B) than its stream twin. -`Xls_ExcelReader_Memory` is ~1% faster than its stream twin (11.031 ms vs. 11.196 ms for `Xls_ExcelReader`), with near-identical allocation (11.87 KB vs. 12.02 KB). In earlier runs the in-memory path was ~33% faster, thanks to `BiffCursor` caching the enclosing contiguous sector run; the stream path has since mostly caught up. +`Xls_ExcelReader_Memory` is ~3% faster than its stream twin (9.080 ms vs. 9.383 ms for `Xls_ExcelReader`), with near-identical allocation (10.39 KB vs. 10.55 KB). In earlier runs the in-memory path was ~33% faster, thanks to `BiffCursor` caching the enclosing contiguous sector run; the stream path has since mostly caught up. ### String-heavy reads @@ -186,22 +186,22 @@ The real-data corpus above is mostly numbers and dates — its shared-string tab | Format | ExcelReader | ExcelReader, prefetch | Sylvan | |---|---:|---:|---:| -| XLSX | 30.00 ms, 2.18 MB | 20.01 ms, 2.21 MB | 176.47 ms, 17.41 MB | -| XLSB | 33.56 ms, 2.18 MB | 19.52 ms, 2.20 MB | 58.80 ms, 17.38 MB | +| XLSX | 30.50 ms, 746.87 KB | 20.47 ms, 783.11 KB | 184.25 ms, 17.41 MB | +| XLSB | 26.40 ms, 746.34 KB | 16.81 ms, 791.60 KB | 60.08 ms, 17.38 MB | -Both formats handle this well: ~5.9x and ~1.8x faster than Sylvan for XLSX and XLSB respectively, at roughly ~8x less memory in both cases, with no garbage collections in either configuration. XLSB's shared-string path previously materialized its table eagerly (~27 MB here); `ParseSharedStreaming` brought it in line with the XLSX streaming/pooling path, cutting allocation by ~12x on this workload. +Both formats handle this well: ~6.0x and ~2.3x faster than Sylvan for XLSX and XLSB respectively, at ~24x less memory in both cases, with no garbage collections in either configuration. XLSB's shared-string path previously materialized its table eagerly (~27 MB here); `ParseSharedStreaming` brought it in line with the XLSX streaming/pooling path. `XlsxSharedStringHotPathBenchmark` splits the XLSX read into stages on the same workbook. The stored rows re-zip it without compression, so they show what parsing costs once inflate is gone: | Stage | Mean | Allocated | |---|---:|---:| -| Whole read | 30.01 ms | 2.18 MB | -| Whole read, prefetch | 19.56 ms | 2.21 MB | -| Shared-string table only (open + first row) | 9.22 ms | 747 KB | -| Whole read, stored | 17.65 ms | 2.18 MB | -| Shared-string table only, stored | 3.34 ms | 746 KB | +| Whole read | 32.20 ms | 747 KB | +| Whole read, prefetch | 19.94 ms | 782 KB | +| Shared-string table only (open + first row) | 9.57 ms | 747 KB | +| Whole read, stored | 18.80 ms | 746 KB | +| Shared-string table only, stored | 3.56 ms | 746 KB | -About 41% of the default read is inflate: without it the whole read drops from 30.01 ms to 17.65 ms, and the shared-string table from 9.22 ms to 3.34 ms. What remains, ~14 ms, is the worksheet scan. +About 42% of the default read is inflate: without it the whole read drops from 32.20 ms to 18.80 ms, and the shared-string table from 9.57 ms to 3.56 ms. What remains, ~15 ms, is the worksheet scan. ### Inflate @@ -209,9 +209,9 @@ About 41% of the default read is inflate: without it the whole read drops from 3 | Entry | `DeflateStream` | `InflateStream` | Ratio | |---|---:|---:|---:| -| XLSX `sheet1.xml` (6.7 MB → 34.4 MB) | 22.244 ms, 280 B | 13.463 ms, 200 B | 0.61 | -| XLSB `sheet1.bin` (3.7 MB → 15.9 MB) | 12.600 ms, 280 B | 9.040 ms, 200 B | 0.72 | -| String-heavy `sharedStrings.xml` | 10.360 ms, 280 B | 5.763 ms, 200 B | 0.56 | +| XLSX `sheet1.xml` (6.7 MB → 34.4 MB) | 22.908 ms, 280 B | 13.537 ms, 200 B | 0.59 | +| XLSB `sheet1.bin` (3.7 MB → 15.9 MB) | 12.977 ms, 280 B | 9.139 ms, 200 B | 0.70 | +| String-heavy `sharedStrings.xml` | 10.721 ms, 280 B | 5.842 ms, 200 B | 0.54 | Without prefetch a read is inflate plus parse, so the faster inflate converts directly; with prefetch the two overlap and the read is bounded by the slower stage, which is why the prefetch columns above moved little. @@ -250,15 +250,15 @@ The encoder's limits are deliberate: | Write | XLSX | vs serial | XLSB | vs serial | |---|---:|---:|---:|---:| -| Serial | 165.08 ms, 16.09 KB | 1.0x | 113.61 ms, 13.42 KB | 1.0x | -| Serial, `PrefetchWrite` | 102.18 ms, 18.35 KB | 1.6x | 62.39 ms, 15.64 KB | 1.8x | -| Parallel, 1 batch in flight | 164.23 ms, 16.15 KB | 1.0x | 114.03 ms, 13.48 KB | 1.0x | -| Parallel, 2 | 84.48 ms, 82.67 KB | 2.0x | 58.53 ms, 81.12 KB | 1.9x | -| Parallel, 4 | 44.27 ms, 79.71 KB | 3.7x | 30.04 ms, 78.45 KB | 3.8x | -| Parallel, 8 | 27.54 ms, 74.09 KB | 6.0x | 19.48 ms, 70.47 KB | 5.8x | -| Parallel, 16 | 21.64 ms, 79.04 KB | 7.6x | 15.37 ms, 67.00 KB | 7.4x | +| Serial | 160.09 ms, 19.44 KB | 1.0x | 110.06 ms, 16.98 KB | 1.0x | +| Serial, `PrefetchWrite` | 99.24 ms, 21.72 KB | 1.6x | 61.54 ms, 19.22 KB | 1.8x | +| Parallel, 1 batch in flight | 174.04 ms, 19.52 KB | 0.9x | 107.89 ms, 17.04 KB | 1.0x | +| Parallel, 2 | 87.64 ms, 86.28 KB | 1.8x | 53.74 ms, 86.70 KB | 2.0x | +| Parallel, 4 | 45.61 ms, 83.28 KB | 3.5x | 28.72 ms, 83.38 KB | 3.8x | +| Parallel, 8 | 28.93 ms, 77.82 KB | 5.5x | 18.66 ms, 74.32 KB | 5.9x | +| Parallel, 16 | 22.07 ms, 81.85 KB | 7.3x | 15.11 ms, 70.65 KB | 7.3x | -It scales because a write is bound by building rows, not by deflate: `PrefetchWrite` already takes deflate off the calling thread and stops at 1.6x to 1.8x, so compressing chunks in parallel would add nothing on top of it, while formatting the rows in parallel does. Past dop 8 the gain shrinks (6.0x to 7.6x for XLSX, 5.8x to 7.4x for XLSB), because the machine has 8 physical cores. None of the rows triggers a garbage collection, and every write allocates under 85 KB for 1,000,000 rows. +It scales because a write is bound by building rows, not by deflate: `PrefetchWrite` already takes deflate off the calling thread and stops at 1.6x to 1.8x, so compressing chunks in parallel would add nothing on top of it, while formatting the rows in parallel does. Past dop 8 the gain shrinks (5.5x to 7.3x for XLSX, 5.9x to 7.3x for XLSB), because the machine has 8 physical cores. None of the rows triggers a garbage collection, and every write allocates under 90 KB for 1,000,000 rows. Each batch is compressed with no reference to the one before it, which costs 0.18% of file size for XLSX (15,481,372 B to 15,508,760 B) and 0.04% for XLSB (16,863,131 B to 16,869,306 B). The process's peak working set rose by 12 MB over the serial write with 4 batches in flight and by 23 MB with 8, about 3 MB per batch, and does not depend on the number of rows. The file sizes and working sets come from a console harness, because BenchmarkDotNet does not report them. @@ -272,30 +272,30 @@ Real-data workbook, per format: | Format | Mean | StdDev | Allocated | |---|---:|---:|---:| -| XLSX | 39.13 ms | 0.08 ms | 16.66 KB | -| XLSM | 39.48 ms | 0.04 ms | 16.66 KB | -| XLSB | 27.93 ms | 0.01 ms | 16.19 KB | -| XLS | 12.73 ms | 0.02 ms | 21.48 KB | -| CSV | 18.95 ms | 0.09 ms | 35.71 MB | +| XLSX | 39.44 ms | 0.78 ms | 16.61 KB | +| XLSM | 39.51 ms | 0.22 ms | 16.61 KB | +| XLSB | 19.77 ms | 0.03 ms | 16.13 KB | +| XLS | 10.29 ms | 0.07 ms | 21.70 KB | +| CSV | 12.85 ms | 0.06 ms | 34.21 MB | Generated 50,000-row XLSX workbook: | Scenario | Mean | StdDev | Allocated | |---|---:|---:|---:| -| Cell-by-cell read, materialized | 8.42 ms | 0.24 ms | 1.58 MB | +| Cell-by-cell read, materialized | 7.03 ms | 0.16 ms | 1.58 MB | String-heavy workbook (65,536 rows, ~190,000 distinct shared strings): | Format | Mean | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |---|---:|---:|---:|---:|---:|---:| -| XLSX | 50.77 ms | 0.30 ms | 1,100.0 | 1,000.0 | 300.0 | 15.25 MB | -| XLSB | 54.01 ms | 0.24 ms | 1,100.0 | 1,000.0 | 300.0 | 15.25 MB | +| XLSX | 48.52 ms | 0.72 ms | 1,181.8 | 1,090.9 | 363.6 | 15.25 MB | +| XLSB | 46.11 ms | 0.34 ms | 1,181.8 | 1,090.9 | 363.6 | 15.25 MB | Reading the allocation columns against the tables above gives the honest shape of the tradeoff: -- **CSV real data:** 35.71 MB materialized against 888 B for the span-based read of the same file. Every CSV field is a distinct string, so nothing dedupes — this is where zero-copy reading earns its keep outright. -- **XLSB real data:** 16.19 KB, essentially cheap. That corpus repeats a small set of values, so the shared-string table dedupes and the reader's string cache materializes each distinct value once. -- **String-heavy XLSX:** 15.25 MB materialized, against **Sylvan's 17.41 MB on the same workload** — doing matched work here, ExcelReader allocates ~12% *less* than Sylvan and is still ~3.5x faster (50.77 ms vs. 176.47 ms). The same allocation holds for XLSB (15.25 MB vs. Sylvan's 17.38 MB, also ~12% less), where matched-work time is ~8% ahead of Sylvan (54.01 ms vs. 58.80 ms). This was previously an inversion (ExcelReader allocated ~1.75x *more* than Sylvan here): the per-reader shared-string dedup cache was an unpresized `Dictionary`, and at ~190,000 distinct values its resize/rehash churn (several of the largest resizes landing on the LOH) accounted for the entire gap — the strings themselves were never the problem, since both readers retain the same ~190,000 distinct instances. Replacing the dictionary with a `string?[]` indexed by shared-string index (sized exactly from the table's known count, no resizing) cut the allocation in half and cut wall-clock time by 14-16% on this benchmark too, since the churn was costing cycles, not just memory. +- **CSV real data:** 34.21 MB materialized against 896 B for the span-based read of the same file. Every CSV field is a distinct string, so nothing dedupes — this is where zero-copy reading earns its keep outright. +- **XLSB real data:** 16.13 KB, essentially cheap. That corpus repeats a small set of values, so the shared-string table dedupes and the reader's string cache materializes each distinct value once. +- **String-heavy XLSX:** 15.25 MB materialized, against **Sylvan's 17.41 MB on the same workload** — doing matched work here, ExcelReader allocates ~12% *less* than Sylvan and is still ~3.8x faster (48.52 ms vs. 184.25 ms). The same allocation holds for XLSB (15.25 MB vs. Sylvan's 17.38 MB, also ~12% less), where matched-work time is ~23% ahead of Sylvan (46.11 ms vs. 60.08 ms). This was previously an inversion (ExcelReader allocated ~1.75x *more* than Sylvan here): the per-reader shared-string dedup cache was an unpresized `Dictionary`, and at ~190,000 distinct values its resize/rehash churn (several of the largest resizes landing on the LOH) accounted for the entire gap — the strings themselves were never the problem, since both readers retain the same ~190,000 distinct instances. Replacing the dictionary with a `string?[]` indexed by shared-string index (sized exactly from the table's known count, no resizing) cut the allocation in half and cut wall-clock time by 14-16% on this benchmark too, since the churn was costing cycles, not just memory. The takeaway is not that one column beats the other: it is that ExcelReader's headline read numbers come from a zero-copy path competitors do not expose, and when it does the same work as them, the gap narrows — and here, with the dedup cache fixed, no longer inverts even at high shared-string cardinality. @@ -305,12 +305,12 @@ The takeaway is not that one column beats the other: it is that ExcelReader's he | Format | Mean | Allocated | |---|---:|---:| -| XLSX | 8.381 ms | 4.02 MB | -| XLSB | 5.838 ms | 4.02 MB | -| XLS | 6.182 ms | 4.31 MB | -| CSV | 4.814 ms | 4.00 MB | +| XLSX | 8.194 ms | 4.02 MB | +| XLSB | 6.053 ms | 4.02 MB | +| XLS | 4.938 ms | 4.32 MB | +| CSV | 4.961 ms | 4.00 MB | -Relative ordering: CSV at the front, XLSB and XLS close behind it, then XLSX — the record-mapping layer adds little overhead over hand-written cell-by-cell writes (XLSB 5.838 ms vs. 5.462 ms, XLSX 8.381 ms vs. 8.191 ms). +Relative ordering: XLS and CSV tied at the front, then XLSB, then XLSX. Against hand-written cell-by-cell writes the record-mapping layer adds nothing for XLSX (8.194 ms vs. 8.213 ms) and ~12% for XLSB (6.053 ms vs. 5.388 ms). ### Native and Arrow string writes @@ -318,12 +318,12 @@ Relative ordering: CSV at the front, XLSB and XLS close behind it, then XLSX — | Scenario | Mean | Allocated | |---|---:|---:| -| Native string columns → XLSX | 14.275 ms | 15.93 KB | -| Native string columns → CSV | 5.525 ms | 416 B | -| Arrow `StringArray` → XLSX | 20.674 ms | 15.94 KB | -| Styled rows → XLSX | 5.817 ms | 16.10 KB | +| Native string columns → XLSX | 19.732 ms | 19.27 KB | +| Native string columns → CSV | 4.460 ms | 424 B | +| Arrow `StringArray` → XLSX | 20.812 ms | 19.32 KB | +| Styled rows → XLSX | 5.260 ms | 19.47 KB | -Text arrives through `IRowWriter.WriteUtf8`, which the XLSX and CSV writers copy through as bytes whenever the text needs no escaping, so none of these allocate per cell: the ~16 KB is per-workbook ZIP and part setup, which CSV does not have. +Text arrives through `IRowWriter.WriteUtf8`, which the XLSX and CSV writers copy through as bytes whenever the text needs no escaping, so none of these allocate per cell: the ~19 KB is per-workbook ZIP and part setup, which CSV does not have. ### Ref struct typed parsing (zero-copy) @@ -331,11 +331,11 @@ A `ref struct` model (see [Parse into a ref struct](../guide/parsing.md#parse-in | Target | Mean | Allocated | |---|---:|---:| -| `class` (`ExcelParser`) | 10.91 ms | 3.87 MB | -| `struct` (`ExcelParser`) | 10.68 ms | 1.58 MB | -| `ref struct` + span binding (`ExcelParser`) | 9.02 ms | 3.98 KB | +| `class` (`ExcelParser`) | 8.88 ms | 3.87 MB | +| `struct` (`ExcelParser`) | 8.98 ms | 1.58 MB | +| `ref struct` + span binding (`ExcelParser`) | 8.15 ms | 4.02 KB | -Parsing into a `ref struct` with a `ReadOnlySpan` text column removes essentially all per-row allocation — ~99.9% less than the `class` baseline — and is ~16% faster than the `struct` target, since there's no per-row `string` allocation for the text column. It is not AOT/trim-safe (reflection-based, same tradeoff as `ExcelParser.FromAttributes`). It can be consumed with `foreach` or `await foreach` but not through `IEnumerable`/`IAsyncEnumerable`/LINQ — a `ref struct` element can't be boxed through those interfaces. +Parsing into a `ref struct` with a `ReadOnlySpan` text column removes essentially all per-row allocation — ~99.9% less than the `class` baseline — and is ~9% faster than the `struct` target, since there's no per-row `string` allocation for the text column. It is not AOT/trim-safe (reflection-based, same tradeoff as `ExcelParser.FromAttributes`). It can be consumed with `foreach` or `await foreach` but not through `IEnumerable`/`IAsyncEnumerable`/LINQ — a `ref struct` element can't be boxed through those interfaces. ### ADO.NET bridge (`IDataReader`) @@ -343,13 +343,13 @@ Parsing into a `ref struct` with a `ReadOnlySpan` text column removes esse | Access | Mean | Ratio | Allocated | |---|---:|---:|---:| -| Raw rows (`row[i].Value`, baseline) | 6.542 ms | 1.00 | 3.48 KB | -| `GetBytes` (text column only) | 6.983 ms | 1.07 | 4.18 KB | -| `GetValue` (every column) | 9.568 ms | 1.46 | 5.01 MB | -| Typed getters (`GetString`/`GetInt32`/`GetDateTime`/`GetDouble`) | 9.357 ms | 1.43 | 1.58 MB | -| `DataTable.Load` | 62.94 ms | 9.62 | 22.84 MB | +| Raw rows (`row[i].Value`, baseline) | 6.139 ms | 1.00 | 3.52 KB | +| `GetBytes` (text column only) | 6.037 ms | 0.99 | 4.68 KB | +| `GetValue` (every column) | 8.473 ms | 1.39 | 5.01 MB | +| Typed getters (`GetString`/`GetInt32`/`GetDateTime`/`GetDouble`) | 8.565 ms | 1.40 | 1.58 MB | +| `DataTable.Load` | 35.01 ms | 5.73 | 16.60 MB | -The bridge costs ~1.4x over reading rows directly. `GetValue` boxes every value (the 5 MB), and the typed getters allocate only the text column's `string`s. `GetBytes` copies UTF-8 without making a `string`, so it allocates nothing per row and costs ~7% over the raw read. `DataTable.Load` is dominated by `DataTable`'s own row storage: the same values cost 9.568 ms to read through `GetValue` alone. +The bridge costs ~1.4x over reading rows directly. `GetValue` boxes every value (the 5 MB), and the typed getters allocate only the text column's `string`s. `GetBytes` copies UTF-8 without making a `string`, so it allocates nothing per row and costs nothing measurable over the raw read (6.037 ms vs. 6.139 ms, against a 0.46 ms standard deviation on the baseline). `DataTable.Load` is dominated by `DataTable`'s own row storage: the same values cost 8.473 ms to read through `GetValue` alone. ### Arrow conversion @@ -357,10 +357,10 @@ The bridge costs ~1.4x over reading rows directly. `GetValue` boxes every value | Scenario | Mean | Allocated | |---|---:|---:| -| CSV, typed, explicit schema | 8.56 ms | 3.13 MB | -| CSV, typed, inferred schema | 7.42 ms | 5.62 MB | -| XLSB, typed, explicit schema | 11.89 ms | 3.13 MB | -| CSV, 8 text columns, explicit schema | 10.61 ms | 6.69 MB | +| CSV, typed, explicit schema | 8.81 ms | 3.13 MB | +| CSV, typed, inferred schema | 7.28 ms | 5.62 MB | +| XLSB, typed, explicit schema | 12.01 ms | 3.13 MB | +| CSV, 8 text columns, explicit schema | 10.24 ms | 6.69 MB | `ToArrowRecordBatch` reads through the same column builder as the native `xl_parse_arrow`, then copies each finished buffer once into the managed array Apache.Arrow takes. Before that (per-cell appenders into Apache.Arrow builders) the same rows took 12.26 ms / 8.13 MB (CSV typed), 16.88 ms / 8.14 MB (XLSB) and 22.31 ms / 16.27 MB (8 text columns). The inferred schema is faster than the explicit one because `Excel.InferSchema` types every CSV column as a string, which skips the int64/timestamp/double conversion; it allocates ~1.8x more. The batch holds every row at once and there is no streaming variant. @@ -372,16 +372,16 @@ On the 65K-row real-data fixtures (14 columns, 5 of them text, one thread), `Exc | Scenario | Mean | Allocated | |---|---:|---:| -| Small, plain | 0.208 ms | 8.72 KB | -| Small, encrypted, stream | 26.133 ms | 37.93 KB | -| Small, encrypted, stream, `VerifyEncryptedIntegrity` | 26.505 ms | 40.05 KB | -| Small, encrypted, in memory | 25.552 ms | 52.98 KB | -| Small, encrypted, open only | 26.099 ms | 32.37 KB | -| 300,000 rows, plain | 37.640 ms | 3.48 KB | -| 300,000 rows, encrypted | 64.103 ms | 29.41 KB | -| 300,000 rows, encrypted, open only | 25.471 ms | 28.72 KB | +| Small, plain | 0.215 ms | 8.72 KB | +| Small, encrypted, stream | 25.094 ms | 38.67 KB | +| Small, encrypted, stream, `VerifyEncryptedIntegrity` | 24.696 ms | 40.81 KB | +| Small, encrypted, in memory | 24.761 ms | 53.75 KB | +| Small, encrypted, open only | 25.578 ms | 33.05 KB | +| 300,000 rows, plain | 33.942 ms | 3.52 KB | +| 300,000 rows, encrypted | 59.446 ms | 29.87 KB | +| 300,000 rows, encrypted, open only | 24.877 ms | 29.16 KB | -Nearly all of the cost is a fixed ~25 ms paid when the workbook is opened: deriving the key through 100,000 SHA-512 iterations, which the format requires so that passwords are slow to brute-force. It does not depend on file size (open-only is 26.10 ms for 15 KB and 25.47 ms for 300,000 rows). After that, decrypting while reading costs ~3% over a plain read: 64.10 ms − 25.47 ms = 38.63 ms, against 37.64 ms unencrypted. On the small fixture, integrity verification and the in-memory path stay within ~3% of the plain encrypted read, inside the spread of the key derivation itself. The fixture is too small to show the extra pass that verification makes over a large stream. +Nearly all of the cost is a fixed ~25 ms paid when the workbook is opened: deriving the key through 100,000 SHA-512 iterations, which the format requires so that passwords are slow to brute-force. It does not depend on file size (open-only is 25.58 ms for 15 KB and 24.88 ms for 300,000 rows). After that, decrypting while reading costs ~2% over a plain read: 59.45 ms − 24.88 ms = 34.57 ms, against 33.94 ms unencrypted. On the small fixture, integrity verification and the in-memory path stay within ~2% of the plain encrypted read, inside the spread of the key derivation itself. The fixture is too small to show the extra pass that verification makes over a large stream. ### Cold start @@ -389,8 +389,8 @@ First use of `ExcelParser.FromAttributes`/`ExcelRecordLayout.FromAttributes`) | 46.05 ms | 20.72 KB | -| First fluent parse (`ExcelParser.Build`) | 41.57 ms | 23.66 KB | -| First fluent parse (`BuildWithAttributeFallback`) | 49.10 ms | 25.78 KB | +| First typed parse (`ExcelParser.FromAttributes`) | 56.25 ms | 20.54 KB | +| First fluent parse (`ExcelParser.Build`) | 44.98 ms | 23.70 KB | +| First fluent parse (`BuildWithAttributeFallback`) | 58.96 ms | 25.60 KB | -`Build` is ~10% faster than the reflection-based parser here; `BuildWithAttributeFallback` is slower than reflection (~7%, with a 0.5 ms standard deviation), since it runs the same `TypeMapper.GetInfo()` path plus the fluent build on top. +`Build` is ~20% faster than the reflection-based parser here; `BuildWithAttributeFallback` is slower than reflection (~5%, with a 0.3 ms standard deviation), since it runs the same `TypeMapper.GetInfo()` path plus the fluent build on top. Run the benchmarks locally: @@ -416,26 +416,26 @@ dotnet run --project tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj | Entry point | Shape handed back | Mean | Managed allocated | |---|---|---:|---:| -| `xl_rows_next` | one row per call, serialized into a caller buffer | 32.06 ms | 3.71 MB | -| `xl_rows_next_cells` | one row per call, cursor-owned cell views, numbers binary | 19.98 ms | 3.71 MB | -| `xl_rows_read_all_blob` | whole sheet in one buffer | 35.27 ms | 21.05 MB | -| `xl_rows_read_all_decoded` | whole sheet as native row structs | 47.12 ms | 5.71 MB | -| `xl_parse_typed` | typed native columns | 27.03 ms | 3.71 MB | -| `xl_parse_arrow` | Arrow C Data Interface arrays | 27.14 ms | 3.72 MB | +| `xl_rows_next` | one row per call, serialized into a caller buffer | 31.65 ms | 3.71 MB | +| `xl_rows_next_cells` | one row per call, cursor-owned cell views, numbers binary | 19.49 ms | 3.71 MB | +| `xl_rows_read_all_blob` | whole sheet in one buffer | 35.24 ms | 21.05 MB | +| `xl_rows_read_all_decoded` | whole sheet as native row structs | 46.44 ms | 5.71 MB | +| `xl_parse_typed` | typed native columns | 25.75 ms | 3.71 MB | +| `xl_parse_arrow` | Arrow C Data Interface arrays | 26.01 ms | 3.72 MB | The allocation column is **managed memory only**. `MemoryDiagnoser` cannot see the native blocks that `xl_rows_read_all_decoded`, `xl_parse_typed` and `xl_parse_arrow` return their data in. -Every cell that crosses the ABI as text costs time: each binary XLSB number is formatted to UTF-8 on the way out. `xl_rows_next_cells` passes numbers as doubles and copies only the text, so it reads the sheet ~38% faster than `xl_rows_next`, and faster than `xl_parse_typed`, which also converts every cell to its column type. `xl_rows_read_all_blob` adds ~10% over `xl_rows_next`, because the whole sheet is buffered before it is copied out. `xl_rows_read_all_decoded` adds ~47%, because it allocates a native block per row. `xl_parse_typed` and `xl_parse_arrow` return columnar native buffers with numbers still binary, so they are the path to use for bulk typed loads; `xl_rows_next_cells` is the one for reading cells as they are. +Every cell that crosses the ABI as text costs time: each binary XLSB number is formatted to UTF-8 on the way out. `xl_rows_next_cells` passes numbers as doubles and copies only the text, so it reads the sheet ~38% faster than `xl_rows_next`, and faster than `xl_parse_typed`, which also converts every cell to its column type. `xl_rows_read_all_blob` adds ~11% over `xl_rows_next`, because the whole sheet is buffered before it is copied out. `xl_rows_read_all_decoded` adds ~47%, because it allocates a native block per row. `xl_parse_typed` and `xl_parse_arrow` return columnar native buffers with numbers still binary, so they are the path to use for bulk typed loads; `xl_rows_next_cells` is the one for reading cells as they are. `ChunkedParseBenchmark` compares `xl_parse_typed` on a whole sheet against draining the same sheet in batches through `xl_typed_reader_open`/`xl_typed_reader_next`. It uses the generated 50,000-row XLSX file, read from disk, with three nullable columns (string, int64, float64): | Mode | Mean | Managed allocated | |---|---:|---:| -| Whole sheet (`xl_parse_typed`) | 8.76 ms | 9.87 KB | -| Batches of 10,000 rows | 8.83 ms | 16.82 KB | -| Batches of 1,000 rows | 8.14 ms | 81.15 KB | +| Whole sheet (`xl_parse_typed`) | 6.97 ms | 10.63 KB | +| Batches of 10,000 rows | 6.68 ms | 18.97 KB | +| Batches of 1,000 rows | 7.00 ms | 97.37 KB | -Batching costs nothing in time: batches of 10,000 rows land within 1% of one whole-sheet table, and batches of 1,000 rows are ~7% faster. A caller can bound its native memory to one batch at a time without paying for it. +Batching costs nothing in time: batches of 10,000 rows are ~4% faster than one whole-sheet table, and batches of 1,000 rows land within 1% of it. A caller can bound its native memory to one batch at a time without paying for it. ### C++ and Rust bindings diff --git a/docs/performance/reader-performance.md b/docs/performance/reader-performance.md index 4ae9e723..12e328e4 100644 --- a/docs/performance/reader-performance.md +++ b/docs/performance/reader-performance.md @@ -1026,7 +1026,7 @@ regenerating it from this tree moves XLSX by ≤0.5% (85.1 against 85.4 ms). `Il is worth ~1%. A trap for whoever regenerates the profile: the ILC publish is incremental and ignores a changed -`.mibc`, producing a byte-identical library. Delete `src/ExcelReader.Native/obj/Release/net10.0//native` +`.mibc`, producing a byte-identical library. Delete `src/ExcelReader.Native/obj/Release/net11.0//native` before publishing. The C++ binding's full XLSX read moved from 93.2 to ~85.4 ms with no native change, from the diff --git a/global.json b/global.json index 4d4dd03d..9858f335 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "10.0.402", + "version": "11.0.100", "rollForward": "latestMinor", "allowPrerelease": false }, diff --git a/python/README.md b/python/README.md index 8fc43d0f..6ef4190b 100644 --- a/python/README.md +++ b/python/README.md @@ -17,7 +17,7 @@ package installs as `excelreader-native` and imports as `excelreader`. ## Install (from source) ```bash -python python/scripts/build_native.py # requires the .NET 10 SDK, once per machine +python python/scripts/build_native.py # requires the .NET 11 SDK, once per machine pip install -e "python[dev]" ``` diff --git a/python/scripts/build_musl_wheel.sh b/python/scripts/build_musl_wheel.sh index 95448cb2..9c7aa98a 100644 --- a/python/scripts/build_musl_wheel.sh +++ b/python/scripts/build_musl_wheel.sh @@ -5,7 +5,7 @@ # Run from the repository root: sh python/scripts/build_musl_wheel.sh set -eu -docker run --rm -v "$PWD:/repo" -w /repo mcr.microsoft.com/dotnet/sdk:10.0-alpine \ +docker run --rm -v "$PWD:/repo" -w /repo mcr.microsoft.com/dotnet/sdk:11.0-alpine \ sh -c 'apk add --no-cache clang build-base zlib-dev python3 && python3 python/scripts/build_native.py' docker run --rm -v "$PWD:/repo" -w /repo python:3.12-alpine \ diff --git a/python/scripts/build_native.py b/python/scripts/build_native.py index 319a3437..ac5f4400 100644 --- a/python/scripts/build_native.py +++ b/python/scripts/build_native.py @@ -49,7 +49,7 @@ def publish_environment() -> dict[str, str]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--rid", default=default_rid(), help="dotnet runtime identifier") - parser.add_argument("--framework", default="net10.0") + parser.add_argument("--framework", default="net11.0") args = parser.parse_args() subprocess.run( diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 171d53ac..edf71b48 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -294,9 +294,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "calamine" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb" +checksum = "f431b67cba6122a1f7d26b7adafc71bbdf07026095fe811f5efc025d8748e1d8" dependencies = [ "atoi_simd", "byteorder", @@ -318,9 +318,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155" dependencies = [ "find-msvc-tools", "shlex", diff --git a/rust/excelreader/Cargo.toml b/rust/excelreader/Cargo.toml index 694cba89..dd62994a 100644 --- a/rust/excelreader/Cargo.toml +++ b/rust/excelreader/Cargo.toml @@ -25,7 +25,7 @@ cc = "1" [dev-dependencies] trybuild = "1" criterion = "0.8" -calamine = { version = "0.37.0", features = ["chrono"] } +calamine = { version = "0.38.0", features = ["chrono"] } rust_xlsxwriter = "0.102.0" [[bench]] diff --git a/rust/excelreader/build.rs b/rust/excelreader/build.rs index 4a628e0a..2181b3b0 100644 --- a/rust/excelreader/build.rs +++ b/rust/excelreader/build.rs @@ -78,11 +78,10 @@ fn main() { } println!("cargo:rustc-link-search=native={}", out_dir.display()); println!("cargo:rustc-link-lib=dylib=excelreader_native"); - if let Some(profile_dir) = out_dir.ancestors().nth(3) { - let deps_dir = profile_dir.join("deps"); - fs::create_dir_all(&deps_dir) - .unwrap_or_else(|e| panic!("failed to create {}: {e}", deps_dir.display())); - let dest = deps_dir.join(&dll_basename); + // cargo puts link-search dirs inside the build dir on PATH for `cargo test`/`run`; an + // EXCELREADER_NATIVE_LIB_DIR outside it isn't, so the DLL goes next to the import lib. + if lib_dir != out_dir { + let dest = out_dir.join(&dll_basename); fs::copy(lib_dir.join(&dll_basename), &dest).unwrap_or_else(|e| { panic!("failed to copy {dll_basename} to {}: {e}", dest.display()) }); diff --git a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt index 2b6ce440..c960b373 100644 --- a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt +++ b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt @@ -2,6 +2,8 @@ ExcelReader.Arrow.ArrowConversionExtensions ExcelReader.Arrow.ArrowReadOptions ExcelReader.Arrow.ArrowReadOptions.ArrowReadOptions() -> void +ExcelReader.Arrow.ArrowReadOptions.DegreeOfParallelism.get -> int +ExcelReader.Arrow.ArrowReadOptions.DegreeOfParallelism.init -> void ExcelReader.Arrow.ArrowReadOptions.HeaderRow.get -> int ExcelReader.Arrow.ArrowReadOptions.HeaderRow.init -> void ExcelReader.Arrow.ArrowReadOptions.Schema.get -> System.Collections.Generic.IReadOnlyList? diff --git a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt index 6114b2ff..7dc5c581 100644 --- a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt @@ -1,3 +1 @@ #nullable enable -ExcelReader.Arrow.ArrowReadOptions.DegreeOfParallelism.get -> int -ExcelReader.Arrow.ArrowReadOptions.DegreeOfParallelism.init -> void diff --git a/src/ExcelReader.Cli/ExcelReader.Cli.csproj b/src/ExcelReader.Cli/ExcelReader.Cli.csproj index d76e2e9a..5b6018f7 100644 --- a/src/ExcelReader.Cli/ExcelReader.Cli.csproj +++ b/src/ExcelReader.Cli/ExcelReader.Cli.csproj @@ -45,7 +45,7 @@ - + all runtime compile; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/ExcelReader.Core/PublicAPI/PublicAPI.Shipped.txt b/src/ExcelReader.Core/PublicAPI/PublicAPI.Shipped.txt index 590a8c4f..6ae27036 100644 --- a/src/ExcelReader.Core/PublicAPI/PublicAPI.Shipped.txt +++ b/src/ExcelReader.Core/PublicAPI/PublicAPI.Shipped.txt @@ -313,6 +313,7 @@ ExcelReader.Core.Reader.Schema.ExcelColumnType.DateColumn = 4 -> ExcelReader.Cor ExcelReader.Core.Reader.Schema.ExcelColumnType.Float64Column = 2 -> ExcelReader.Core.Reader.Schema.ExcelColumnType ExcelReader.Core.Reader.Schema.ExcelColumnType.Int64Column = 1 -> ExcelReader.Core.Reader.Schema.ExcelColumnType ExcelReader.Core.Reader.Schema.ExcelColumnType.StringColumn = 0 -> ExcelReader.Core.Reader.Schema.ExcelColumnType +ExcelReader.Core.Reader.Schema.ExcelColumnType.StringDictionaryColumn = 7 -> ExcelReader.Core.Reader.Schema.ExcelColumnType ExcelReader.Core.Reader.Schema.ExcelColumnType.TimeColumn = 5 -> ExcelReader.Core.Reader.Schema.ExcelColumnType ExcelReader.Core.Reader.Schema.ExcelColumnType.TimestampColumn = 6 -> ExcelReader.Core.Reader.Schema.ExcelColumnType ExcelReader.Core.Reader.Schema.SchemaInferenceOptions @@ -515,6 +516,7 @@ ExcelReader.Core.Writer.Csv.CsvSheetWriter.End() -> void ExcelReader.Core.Writer.Csv.CsvSheetWriter.EndAsync(System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask ExcelReader.Core.Writer.Csv.CsvSheetWriter.SetColumnStyle(int columnIndex, int styleId) -> void ExcelReader.Core.Writer.Csv.CsvSheetWriter.SetColumnWidth(int columnIndex, double width) -> void +ExcelReader.Core.Writer.Csv.CsvSheetWriter.SetShowGridLines(bool show) -> void ExcelReader.Core.Writer.Csv.CsvSheetWriter.StartRow() -> ExcelReader.Core.Writer.Csv.CsvRowWriter! ExcelReader.Core.Writer.Csv.CsvSheetWriter.StartRow(int styleId) -> ExcelReader.Core.Writer.Csv.CsvRowWriter! ExcelReader.Core.Writer.Csv.CsvSheetWriter.StartRow(int styleId, double height) -> ExcelReader.Core.Writer.Csv.CsvRowWriter! @@ -668,6 +670,7 @@ ExcelReader.Core.Writer.ISheetWriter.End() -> void ExcelReader.Core.Writer.ISheetWriter.EndAsync(System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask ExcelReader.Core.Writer.ISheetWriter.SetColumnStyle(int columnIndex, int styleId) -> void ExcelReader.Core.Writer.ISheetWriter.SetColumnWidth(int columnIndex, double width) -> void +ExcelReader.Core.Writer.ISheetWriter.SetShowGridLines(bool show) -> void ExcelReader.Core.Writer.ISheetWriter.StartRow() -> TRow ExcelReader.Core.Writer.ISheetWriter.StartRow(int styleId) -> TRow ExcelReader.Core.Writer.ISheetWriter.StartRow(int styleId, double height) -> TRow @@ -718,6 +721,7 @@ ExcelReader.Core.Writer.Xls.XlsSheetWriter.End() -> void ExcelReader.Core.Writer.Xls.XlsSheetWriter.EndAsync(System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask ExcelReader.Core.Writer.Xls.XlsSheetWriter.SetColumnStyle(int columnIndex, int styleId) -> void ExcelReader.Core.Writer.Xls.XlsSheetWriter.SetColumnWidth(int columnIndex, double width) -> void +ExcelReader.Core.Writer.Xls.XlsSheetWriter.SetShowGridLines(bool show) -> void ExcelReader.Core.Writer.Xls.XlsSheetWriter.StartRow() -> ExcelReader.Core.Writer.Xls.XlsRowWriter! ExcelReader.Core.Writer.Xls.XlsSheetWriter.StartRow(int styleId) -> ExcelReader.Core.Writer.Xls.XlsRowWriter! ExcelReader.Core.Writer.Xls.XlsSheetWriter.StartRow(int styleId, double height) -> ExcelReader.Core.Writer.Xls.XlsRowWriter! @@ -770,6 +774,7 @@ ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.EndAsync(System.Threading.Cancellat ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.EndTable() -> void ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.SetColumnStyle(int columnIndex, int styleId) -> void ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.SetColumnWidth(int columnIndex, double width) -> void +ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.SetShowGridLines(bool show) -> void ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.StartRow() -> ExcelReader.Core.Writer.Xlsb.XlsbRowWriter! ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.StartRow(int styleId) -> ExcelReader.Core.Writer.Xlsb.XlsbRowWriter! ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.StartRow(int styleId, double height) -> ExcelReader.Core.Writer.Xlsb.XlsbRowWriter! @@ -826,6 +831,7 @@ ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.EndAsync(System.Threading.Cancellat ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.EndTable() -> void ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.SetColumnStyle(int columnIndex, int styleId) -> void ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.SetColumnWidth(int columnIndex, double width) -> void +ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.SetShowGridLines(bool show) -> void ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.StartRow() -> ExcelReader.Core.Writer.Xlsx.XlsxRowWriter! ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.StartRow(int styleId) -> ExcelReader.Core.Writer.Xlsx.XlsxRowWriter! ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.StartRow(int styleId, double height) -> ExcelReader.Core.Writer.Xlsx.XlsxRowWriter! diff --git a/src/ExcelReader.Core/PublicAPI/PublicAPI.Unshipped.txt b/src/ExcelReader.Core/PublicAPI/PublicAPI.Unshipped.txt index fd11c8c2..7dc5c581 100644 --- a/src/ExcelReader.Core/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/ExcelReader.Core/PublicAPI/PublicAPI.Unshipped.txt @@ -1,7 +1 @@ #nullable enable -ExcelReader.Core.Writer.Csv.CsvSheetWriter.SetShowGridLines(bool show) -> void -ExcelReader.Core.Writer.ISheetWriter.SetShowGridLines(bool show) -> void -ExcelReader.Core.Writer.Xls.XlsSheetWriter.SetShowGridLines(bool show) -> void -ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter.SetShowGridLines(bool show) -> void -ExcelReader.Core.Writer.Xlsx.XlsxSheetWriter.SetShowGridLines(bool show) -> void -ExcelReader.Core.Reader.Schema.ExcelColumnType.StringDictionaryColumn = 7 -> ExcelReader.Core.Reader.Schema.ExcelColumnType diff --git a/src/ExcelReader.Core/Reader/Internal/Utf8Strings.cs b/src/ExcelReader.Core/Reader/Internal/Utf8Strings.cs index a6d4c05e..e95161a2 100644 --- a/src/ExcelReader.Core/Reader/Internal/Utf8Strings.cs +++ b/src/ExcelReader.Core/Reader/Internal/Utf8Strings.cs @@ -1,18 +1,121 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Text; namespace ExcelReader.Core.Reader.Internal { internal static class Utf8Strings { + private const ulong HighBits = 0x8080808080808080UL; + + private static readonly string[] SingleAscii = CreateSingleAscii(); + /// Materializes UTF-8 cell text. ASCII widens straight into the new string; Encoding.UTF8 /// counts characters in one pass and transcodes in a second, which dominates short cell values. internal static string Create(ReadOnlySpan utf8) { + if (BitConverter.IsLittleEndian && (uint)(utf8.Length - 1) < 15u) + { + return CreateShort(utf8); + } if (!Ascii.IsValid(utf8)) { return Encoding.UTF8.GetString(utf8); } return string.Create(utf8.Length, utf8, static (chars, ascii) => Ascii.ToUtf16(ascii, chars, out _)); } + + // Loads the first and last word of a 1..15-byte value. The two may overlap; every load stays + // inside the source and every store inside the new string. + private static string CreateShort(ReadOnlySpan utf8) + { + int length = utf8.Length; + ref byte source = ref MemoryMarshal.GetReference(utf8); + ulong head; + ulong tail; + if (length >= 8) + { + head = Unsafe.ReadUnaligned(ref source); + tail = Unsafe.ReadUnaligned(ref Unsafe.Add(ref source, length - 8)); + if (!IsAscii(head | tail)) + { + return Encoding.UTF8.GetString(utf8); + } + return string.Create(length, (head, tail), Widen8); + } + if (length >= 4) + { + head = Unsafe.ReadUnaligned(ref source); + tail = Unsafe.ReadUnaligned(ref Unsafe.Add(ref source, length - 4)); + if (!IsAscii(head | tail)) + { + return Encoding.UTF8.GetString(utf8); + } + return string.Create(length, (head, tail), Widen4); + } + if (length >= 2) + { + head = Unsafe.ReadUnaligned(ref source); + tail = Unsafe.ReadUnaligned(ref Unsafe.Add(ref source, length - 2)); + if (!IsAscii(head | tail)) + { + return Encoding.UTF8.GetString(utf8); + } + return string.Create(length, (head, tail), Widen2); + } + if (source < 0x80) + { + return SingleAscii[source]; + } + return Encoding.UTF8.GetString(utf8); + } + + private static bool IsAscii(ulong bytes) + { + return (bytes & HighBits) == 0; + } + + private static void Widen8(Span chars, (ulong Head, ulong Tail) words) + { + ref byte destination = ref Unsafe.As(ref MemoryMarshal.GetReference(chars)); + Vector128.WidenLower(Vector128.CreateScalarUnsafe(words.Head).AsByte()).AsByte().StoreUnsafe(ref destination); + Vector128.WidenLower(Vector128.CreateScalarUnsafe(words.Tail).AsByte()).AsByte().StoreUnsafe(ref Unsafe.Add(ref destination, (chars.Length - 8) * 2)); + } + + private static void Widen4(Span chars, (ulong Head, ulong Tail) words) + { + ref byte destination = ref Unsafe.As(ref MemoryMarshal.GetReference(chars)); + Unsafe.WriteUnaligned(ref destination, SpreadFour(words.Head)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, (chars.Length - 4) * 2), SpreadFour(words.Tail)); + } + + private static void Widen2(Span chars, (ulong Head, ulong Tail) words) + { + ref byte destination = ref Unsafe.As(ref MemoryMarshal.GetReference(chars)); + Unsafe.WriteUnaligned(ref destination, SpreadTwo((uint)words.Head)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, (chars.Length - 2) * 2), SpreadTwo((uint)words.Tail)); + } + + private static ulong SpreadFour(ulong bytes) + { + bytes = (bytes | (bytes << 16)) & 0x0000FFFF0000FFFFUL; + return (bytes | (bytes << 8)) & 0x00FF00FF00FF00FFUL; + } + + private static uint SpreadTwo(uint bytes) + { + return (bytes | (bytes << 8)) & 0x00FF00FFu; + } + + private static string[] CreateSingleAscii() + { + string[] table = new string[128]; + for (int c = 0; c < table.Length; c++) + { + table[c] = ((char)c).ToString(); + } + return table; + } } } diff --git a/src/ExcelReader.Core/Reader/Schema/SchemaInference.cs b/src/ExcelReader.Core/Reader/Schema/SchemaInference.cs index 0aab3ca5..21111b6e 100644 --- a/src/ExcelReader.Core/Reader/Schema/SchemaInference.cs +++ b/src/ExcelReader.Core/Reader/Schema/SchemaInference.cs @@ -111,6 +111,7 @@ private static int SampleDataRows(IExcelRowEnumerator rows, int sampleSize, bool return dataRowCount; } + [SuppressMessage("Performance", "CA1517", Justification = "False positive: the foreach ref loop writes through the span.")] private static void MarkSparseColumnsNullable(Span stats, int dataRowCount) { foreach (ref ColumnStat stat in stats) diff --git a/src/ExcelReader.Core/Reader/Xls/BiffCursor.cs b/src/ExcelReader.Core/Reader/Xls/BiffCursor.cs index 89ce8d92..f8d07419 100644 --- a/src/ExcelReader.Core/Reader/Xls/BiffCursor.cs +++ b/src/ExcelReader.Core/Reader/Xls/BiffCursor.cs @@ -78,6 +78,29 @@ internal bool TryReadRecord(out int id, out ReadOnlySpan data) return true; } + /// The stream bytes from that are already addressable without a copy. + internal ReadOnlySpan LoadedSpan() + { + long pos = Position; + if (pos >= Length) + { + return default; + } + if (_kind == WorkbookStream.SourceKind.Contiguous) + { + return _wb.Memory(pos, (int)(Length - pos)); + } + if (_kind == WorkbookStream.SourceKind.Chained && pos >= _runStart && pos < _runEnd) + { + return _file.AsSpan(_fileBase + (int)(_runBufferOffset + pos - _runStart), (int)(_runEnd - pos)); + } + if (_kind == WorkbookStream.SourceKind.Streamed && pos >= _windowStart && pos < _windowEnd) + { + return _sector.AsSpan((int)(pos - _windowStart), (int)(Math.Min(_windowEnd, Length) - pos)); + } + return default; + } + private ReadOnlySpan ReadSpan(long pos, int len) { if (_kind == WorkbookStream.SourceKind.Contiguous) diff --git a/src/ExcelReader.Core/Reader/Xls/XlsWorkbook.Enumerator.cs b/src/ExcelReader.Core/Reader/Xls/XlsWorkbook.Enumerator.cs index b984f822..3ba2bc2c 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsWorkbook.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsWorkbook.Enumerator.cs @@ -63,6 +63,10 @@ private bool MoveNextCore() BiffCursor cursor = _cursor; while (true) { + if (_row >= 0) + { + DrainCells(cursor); + } long recordStart = cursor.Position; if (!cursor.TryReadRecord(out int id, out ReadOnlySpan data)) { @@ -154,6 +158,91 @@ private bool ReadRecord(BiffCursor cursor, long recordStart, int id, ReadOnlySpa } } + // Writes the current row's number and shared-string records straight into the cell array. + // Stops at the first record it is not sure about and leaves it to ReadRecord. + private void DrainCells(BiffCursor cursor) + { + ReadOnlySpan data = cursor.LoadedSpan(); + CellDesc[] cells = _acc.RawCells; + int count = _acc.Count; + int lastCol = _acc.LastColumn; + bool sorted = true; + int position = 0; + while (position + 14 <= data.Length) + { + int id = ReadU16(data, position); + int length = ReadU16(data, position + 2); + if (length < 10 || position + 4 + length > data.Length || ReadU16(data, position + 4) != _row) + { + break; + } + ReadOnlySpan payload = data.Slice(position + 4, length); + int col = ReadU16(payload, 2); + int added = id == Rec.MulRk ? (length - 6) / 6 : 1; + if (added == 0 || col + added > ExcelLimits.MaxColumns || count + added > cells.Length || !TryDrainCell(id, payload, cells.AsSpan(count, added))) + { + break; + } + sorted &= col >= lastCol; + lastCol = col + added - 1; + count += added; + position += 4 + length; + } + _acc.Commit(count, lastCol, sorted); + cursor.Position += position; + } + + private bool TryDrainCell(int id, ReadOnlySpan payload, Span destination) + { + int col = ReadU16(payload, 2); + int style = ReadU16(payload, 4); + switch (id) + { + case Rec.LabelSst: + var (start, len, sharedIndex) = _reader.SharedAt(ReadI32(payload, 6)); + destination[0] = new CellDesc + { + Column = col, + Start = start, + Length = len, + Type = CellType.ExcelString, + Style = style, + Source = CellValueSource.Shared, + SharedIndex = sharedIndex, + }; + return true; + case Rec.Number when payload.Length >= 14: + destination[0] = NumberCell(col, style, BinaryPrimitives.ReadDoubleLittleEndian(payload.Slice(6, 8))); + return true; + case Rec.Rk: + destination[0] = NumberCell(col, style, Rk(ReadU32(payload, 6))); + return true; + case Rec.MulRk: + for (int i = 0; i < destination.Length; i++) + { + destination[i] = NumberCell(col + i, ReadU16(payload, 4 + (i * 6)), Rk(ReadU32(payload, 6 + (i * 6)))); + } + return true; + default: + return false; + } + } + + private CellDesc NumberCell(int col, int style, double value) + { + return new CellDesc + { + Column = col, + Start = _acc.ValueLength, + Type = _reader.IsDateStyle(style) ? CellType.Date : CellType.Number, + Style = style, + Source = CellValueSource.RowValues, + Number = value, + HasNumber = true, + SharedIndex = -1, + }; + } + private void ReadBof(BiffCursor cursor, ReadOnlySpan data) { if (_sawBof) diff --git a/src/ExcelReader.Native/ExcelReader.Native.csproj b/src/ExcelReader.Native/ExcelReader.Native.csproj index f35829e1..dff0f50b 100644 --- a/src/ExcelReader.Native/ExcelReader.Native.csproj +++ b/src/ExcelReader.Native/ExcelReader.Native.csproj @@ -1,7 +1,10 @@ - net10.0 + + net10.0;net11.0 true false true Shared + + false false true diff --git a/src/ExcelReader.Native/README.md b/src/ExcelReader.Native/README.md index 1beb4d51..1ea76529 100644 --- a/src/ExcelReader.Native/README.md +++ b/src/ExcelReader.Native/README.md @@ -8,9 +8,9 @@ NativeAOT shared library exposing ExcelReader's readers over a C ABI, so non-.NE ## Build - dotnet publish src/ExcelReader.Native/ExcelReader.Native.csproj -c Release -f net10.0 -r + dotnet publish src/ExcelReader.Native/ExcelReader.Native.csproj -c Release -f net11.0 -r -Output lands in `bin/Release/net10.0//publish/ExcelReader.Native.{dll,so,dylib}`. +Output lands in `bin/Release/net11.0//publish/ExcelReader.Native.{dll,so,dylib}`. ## Static PGO profile @@ -26,14 +26,14 @@ example) silently drops its profile. That happened after the namespace refactor look for `; No PGO data` in the listing header. `tests/ExcelReader.NativePgoTrainer` runs the implementations of `xl_parse_typed`, `xl_parse_arrow` (with `XL_T_STRING_DICT` columns), `xl_rows_next`, -`xl_rows_next_view`, `xl_rows_next_cells` and `xl_rows_foreach` over the 65K benchmark fixtures in XLSX, XLSB and CSV, and writes each parsed +`xl_rows_next_view`, `xl_rows_next_cells` and `xl_rows_foreach` over the 65K benchmark fixtures in XLSX, XLSB, CSV and XLS, and writes each parsed table back out with `xl_write_typed_to_memory` in XLSX, XLSB and CSV, serially and in parallel. Record it and convert the trace (needs `dotnet-trace` and `dotnet-pgo`): dotnet build tests/ExcelReader.NativePgoTrainer -c Release DOTNET_TieredPGO=1 DOTNET_ReadyToRun=0 DOTNET_TC_QuickJitForLoops=1 \ dotnet-trace collect --providers Microsoft-Windows-DotNETRuntime:0x1F000080018:5 -o pgo.nettrace \ - -- "$PWD/tests/ExcelReader.NativePgoTrainer/bin/Release/net10.0/ExcelReader.NativePgoTrainer" + -- "$PWD/tests/ExcelReader.NativePgoTrainer/bin/Release/net11.0/ExcelReader.NativePgoTrainer" dotnet-pgo create-mibc -t pgo.nettrace -o src/ExcelReader.Native/pgo/excelreader.mibc Until 2026-10 the profile excluded `Xlsb|Biff12`: the old XLSB cell loop got 3-22% slower with its diff --git a/src/ExcelReader.Native/pgo/excelreader.mibc b/src/ExcelReader.Native/pgo/excelreader.mibc index 64414bb5..f5cd1fb1 100644 Binary files a/src/ExcelReader.Native/pgo/excelreader.mibc and b/src/ExcelReader.Native/pgo/excelreader.mibc differ diff --git a/tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj b/tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj index 2771abb0..fd1c1dfc 100644 --- a/tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj +++ b/tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj @@ -26,8 +26,8 @@ - - + + @@ -36,8 +36,7 @@ - + diff --git a/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs b/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs index e533240c..e9485a07 100644 --- a/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs @@ -58,7 +58,7 @@ public long ExcelParserSync() public long ExcelParserSyncInterned() { using var ms = new MemoryStream(_workbook, writable: false); - using var reader = Excel.FromXlsx(ms, options: new ExcelReaderOptions { InternStrings = true }); + using var reader = Excel.FromXlsx(ms, options: new Core.Reader.ExcelReaderOptions { InternStrings = true }); long acc = 0; foreach (Record rec in ExcelParser.FromAttributes().Parse(reader.FirstSheet)) { @@ -149,7 +149,8 @@ public async Task ExcelParserXlsbAsync() [Benchmark] public long OfficeIMO() { - using var reader = ExcelDocument.OpenDataReader(_workbook, new ExcelReadOptions { HasHeaderRow = true }); + using var ms = new MemoryStream(_workbook, writable: false); + using var reader = ExcelDocument.OpenReader(ms, new OfficeIMO.Excel.ExcelReaderOptions { HasHeaderRow = true }); long acc = 0; while (reader.Read()) { diff --git a/tests/ExcelReader.Benchmarks/Read/ReadBenchmark.cs b/tests/ExcelReader.Benchmarks/Read/ReadBenchmark.cs index d865c667..e6a58726 100644 --- a/tests/ExcelReader.Benchmarks/Read/ReadBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Read/ReadBenchmark.cs @@ -103,14 +103,16 @@ public long ExcelDataReaderXlsb() [Benchmark] public long OfficeIMO() { - using var reader = ExcelDocument.OpenDataReader(_workbook, new ExcelReadOptions { HasHeaderRow = false }); + using var ms = new MemoryStream(_workbook, writable: false); + using var reader = ExcelDocument.OpenReader(ms, new OfficeIMO.Excel.ExcelReaderOptions { HasHeaderRow = false }); return AccumulateDataReader(reader); } [Benchmark] public long OfficeIMOXlsb() { - using var reader = ExcelDocument.OpenDataReader(_xlsbWorkbook, new ExcelReadOptions { HasHeaderRow = false }); + using var ms = new MemoryStream(_xlsbWorkbook, writable: false); + using var reader = ExcelDocument.OpenReader(ms, new OfficeIMO.Excel.ExcelReaderOptions { HasHeaderRow = false }); return AccumulateDataReader(reader); } } diff --git a/tests/ExcelReader.Benchmarks/Read/XlsReadBenchmark.cs b/tests/ExcelReader.Benchmarks/Read/XlsReadBenchmark.cs index c141ded6..0fbdd2c5 100644 --- a/tests/ExcelReader.Benchmarks/Read/XlsReadBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Read/XlsReadBenchmark.cs @@ -62,7 +62,8 @@ public long ExcelDataReader() [Benchmark] public long OfficeIMO() { - using var reader = global::OfficeIMO.Excel.ExcelDocument.OpenDataReader(_workbook, new OfficeIMO.Excel.ExcelReadOptions { HasHeaderRow = false }); + using var ms = new MemoryStream(_workbook, writable: false); + using var reader = global::OfficeIMO.Excel.ExcelDocument.OpenReader(ms, new OfficeIMO.Excel.ExcelReaderOptions { HasHeaderRow = false }); return AccumulateDataReader(reader); } } diff --git a/tests/ExcelReader.Benchmarks/Write/WriteBenchmark.cs b/tests/ExcelReader.Benchmarks/Write/WriteBenchmark.cs index 00a08b99..289c346b 100644 --- a/tests/ExcelReader.Benchmarks/Write/WriteBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Write/WriteBenchmark.cs @@ -177,13 +177,16 @@ public async Task ExcelReaderXlsbWriterPrefetch() public long OfficeIMO() { using var ms = new MemoryStream(4 * 1024 * 1024); - ExcelDocument.WriteRows(ms, _records, ["Name", "Id", "Date", "Value"], static (row, rec) => + using (ExcelWorkbookWriter writer = ExcelDocument.CreateWriter(ms, leaveOpen: true)) { - row.Write(rec.Name); - row.Write(rec.Id); - row.Write(rec.Date); - row.Write(rec.Value); - }); + writer.WriteTable("S1", _records, ["Name", "Id", "Date", "Value"], static (row, rec) => + { + row.Write(rec.Name); + row.Write(rec.Id); + row.Write(rec.Date); + row.Write(rec.Value); + }); + } return ms.Length; } diff --git a/tests/ExcelReader.NativePgoTrainer/ExcelReader.NativePgoTrainer.csproj b/tests/ExcelReader.NativePgoTrainer/ExcelReader.NativePgoTrainer.csproj index 8f2f4e73..6b11c5c0 100644 --- a/tests/ExcelReader.NativePgoTrainer/ExcelReader.NativePgoTrainer.csproj +++ b/tests/ExcelReader.NativePgoTrainer/ExcelReader.NativePgoTrainer.csproj @@ -2,7 +2,7 @@ Exe - net10.0 + net11.0 false true