diff --git a/README.md b/README.md index 760d3f18..8e81132a 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,16 @@ let markdown = anydoc::to_markdown_bytes(&bytes, anydoc::Format::Csv)?; let document = anydoc::to_document(&bytes, None)?; ``` +### Spreadsheet provenance + +The structured model returned for `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, and `.ods` +tables keeps worksheet identity and source coordinates. `Table::source` carries +the zero-based worksheet index and name plus an inclusive source range; each +spreadsheet origin `Cell::source` carries its own inclusive range. Coordinates +are zero-based, so `C3` is `{ row: 2, column: 2 }`. Merged origins retain their +full source rectangle even when hidden rows or columns change the normalized +grid. Non-spreadsheet tables and CSV keep these optional fields unset. + ## OCR anydoc reads text-based PDFs locally but does no OCR, so a PDF with scanned or image-only pages fails with `NeedsOcr`. Opt in and those documents go to [Firecrawl Parse](https://firecrawl.dev/parse), which OCRs them and returns the same Markdown. No signup needed; set `FIRECRAWL_API_KEY` for higher limits. diff --git a/docs/spreadsheet-provenance-plan.md b/docs/spreadsheet-provenance-plan.md new file mode 100644 index 00000000..ff21d373 --- /dev/null +++ b/docs/spreadsheet-provenance-plan.md @@ -0,0 +1,173 @@ +# Spreadsheet provenance implementation plan + +This plan implements `docs/spreadsheet-provenance-spec.md` on the +`feat/sheet-provenance` branch. + +## Allowed scope + +Modify only the shared document model, spreadsheet parsers, ODS parser, +bindings, generated binding declarations, relevant tests, and the two +spreadsheet provenance documents. + +Do not change Markdown rendering, format detection, hidden-content policy, +resource limits, unrelated parsers, the CSR project, or dependencies unless a +test proves a narrowly scoped change is required. + +## Commit plan + +### Commit 1 — public model and source-coordinate primitives + +Files: + +- `src/model/source.rs` (new); +- `src/model/mod.rs`; +- `src/model/table.rs`; +- model and renderer construction tests that use explicit struct literals. + +Deliverables: + +- zero-based inclusive `SpreadsheetCoordinate` and `SpreadsheetRange`; +- table-level `SpreadsheetSource` containing original sheet index, name, and + bounding range; +- optional `Table.source` and `Cell.source` fields; +- constructors/defaults preserve `None` for non-spreadsheet tables; +- no renderer behavior change. + +### Commit 2 — Excel provenance through the shared grid path + +Files: + +- `src/formats/sheet/xlsx.rs`; +- `src/formats/sheet/xlsb.rs`; +- `src/formats/sheet/xls.rs`; +- shared spreadsheet tests as needed. + +Deliverables: + +- carry source sheet identity before hidden-sheet filtering; +- attach original one-cell ranges to ordinary and generated empty cells; +- attach original merge rectangles to merged origins; +- compute table source ranges from original coordinates; +- preserve current normalized grid and Markdown output. + +### Commit 3 — ODS provenance with repeat-aware cursors + +Files: + +- `src/formats/odf/table.rs`; +- `src/formats/odf/mod.rs` only if sheet-order context must be threaded there; +- ODS parser tests. + +Deliverables: + +- source row/column cursors independent from normalized `GridBuilder` cursors; +- correct ranges for repeated rows, repeated columns, covered cells, and + row/column spans; +- source range for each returned ODS table; +- existing repeat and expansion limits unchanged. + +### Commit 4 — bindings, declarations, fixtures, and regression coverage + +Files: + +- `node/src/document.rs`, `node/index.d.ts`; +- `python/src/document.rs`, `python/anydoc/_anydoc.pyi`; +- `wasm/src/document.rs`, `wasm/src/typescript.rs`; +- binding tests and any generated declaration updates; +- README/API notes if the existing project convention requires them. + +Deliverables: + +- equivalent Rust/Node/Python/WASM shapes; +- optional provenance is absent/null according to each binding's existing + convention for non-spreadsheet documents; +- binding smoke tests for spreadsheet and DOCX/CSV cases; +- complete diff and compatibility review. + +## Fixture design + +The core parser tests already use compact in-module builders instead of +checking binary fixtures into the repository. Extend that pattern so each +format tests its real reader and the shared model contract. + +### XLSX fixture + +Build a workbook with: + +- visible sheet `Data Sheet` containing `C3 = "value"`; +- another visible sheet with a different name; +- a hidden sheet between them to verify original sheet indices; +- a visible sheet region starting at `D11:E12`; +- a merge such as `F1:G2` and a hidden row or column intersecting a merge. + +Assertions: + +- table source has the original sheet name/index and inclusive bounding range; +- the normalized first cell still has its original source coordinate; +- merged origin retains `F1:G2` even if normalized span changes; +- hidden sheets remain omitted and do not change source indices. + +### XLSB fixture + +Reuse the existing synthetic OPC workbook builder and binary record helpers. +Create the same logical workbook cases as XLSX, using `BrtBundleSh`, +`BrtRowHdr`, cell records, and `BrtMergeCell`. + +Assertions must compare the provenance shape to the XLSX expectations without +assuming the container's internal part names. + +### XLS fixture + +Reuse the existing OLE/BIFF builder and records. Create visible and hidden +`BOUNDSHEET` entries, BIFF cell records at non-zero row/column positions, and a +`MERGEDCELLS` rectangle. + +Assertions must verify BIFF row/column values are retained after the shared +grid builder crops the table. + +### ODS fixture + +Build a minimal `content.xml` with: + +- a table named `Data Sheet`; +- leading empty cells before a populated cell; +- `number-columns-repeated` for a repeated value; +- `number-rows-repeated` for repeated rows; +- a row/column span and explicit covered cells; +- a second table to verify table order and identity. + +Assertions must verify that repeat expansion receives distinct source +coordinates, spans retain their full source ranges, and normalized covered +slots still point to the correct origin. + +### Binding fixture policy + +Bindings should consume the core model fixtures or construct the smallest +in-memory documents possible. They must assert field names and value shapes, +not duplicate parser logic. + +## Verification commands + +Run, when the required toolchains are available: + +```text +cargo fmt --all -- --check +cargo test --workspace +npm test --prefix node +python -m pytest python/tests +``` + +If a binding requires a generated/native artifact or a toolchain is missing, +record the exact command and environment limitation rather than treating the +unrun test as passing. + +## Stop conditions + +Pause implementation and revisit the Spec if any of these occur: + +- source coordinates cannot be defined consistently across a format; +- preserving provenance requires changing Markdown output; +- hidden-sheet behavior would expose previously omitted content; +- a new dependency appears necessary; +- a binding cannot represent the core model without a format-specific shape; +- a public API compatibility issue requires a different model design. diff --git a/docs/spreadsheet-provenance-spec.md b/docs/spreadsheet-provenance-spec.md new file mode 100644 index 00000000..95a5d0b1 --- /dev/null +++ b/docs/spreadsheet-provenance-spec.md @@ -0,0 +1,236 @@ +# Spreadsheet provenance in the document model + +Status: proposed implementation spec + +Related issue: [Expose worksheet identity and source coordinates in `to_document()` #10](https://github.com/firecrawl/anydoc/issues/10) + +## Problem + +The spreadsheet readers already know the source worksheet and the original row +and column of each populated cell. During canonical grid construction, however, +hidden rows and columns are removed, merged regions are remapped, and the +resulting `Table` only exposes normalized grid coordinates. A caller therefore +cannot map `table.grid[0][0]` back to a worksheet cell such as `Data Sheet!C3`. + +The worksheet name is also currently represented only by a Markdown heading +when a workbook has more than one visible sheet. This makes worksheet identity +dependent on the rendering path and loses it for single-sheet workbooks. + +## Scope + +This change applies to the structured document model returned by +`to_document()`. It covers all spreadsheet containers currently parsed by +anydoc: + +- `.xlsx` and `.xlsm` (SpreadsheetML); +- `.xlsb` (binary SpreadsheetML); +- `.xls` (BIFF/OLE); +- `.ods` (OpenDocument Spreadsheet). + +CSV is intentionally out of scope: it has no worksheet identity or workbook +coordinate system. Markdown rendering, format detection, hidden-sheet policy, +and existing resource limits are otherwise unchanged. + +## Goals + +1. Preserve worksheet identity independently of Markdown rendering. +2. Give every returned spreadsheet origin cell an exact source range. +3. Keep normalized `Table.grid` behavior unchanged for existing consumers. +4. Expose the same provenance semantics through Rust, Node.js, Python, and + WASM bindings. +5. Preserve provenance through cropping, hidden row/column filtering, merged + cells, and ODS repeat expansion. + +## Non-goals + +- Preserve formulas, comments, styles, drawings, charts, or other spreadsheet + features that are not currently represented by the document model. +- Change the Markdown output to include sheet names or coordinates. +- Add file paths or file hashes to the model. `to_document()` accepts bytes and + does not have a reliable source path. +- Add provenance for non-spreadsheet tables in this change. + +## Coordinate contract + +Coordinates are zero-based and ranges are inclusive at both ends. + +```rust +SpreadsheetCoordinate { row: 2, column: 2 } // C3 +SpreadsheetRange { + start: SpreadsheetCoordinate { row: 2, column: 2 }, + end: SpreadsheetCoordinate { row: 2, column: 2 }, +} // C3:C3 +``` + +`sheet_index` is the zero-based position in the source workbook/table order, +including hidden Excel sheets. Hidden sheets remain omitted from the returned +document exactly as they are today; their presence must not renumber the +visible sheets that follow them. + +The table range is the smallest source-coordinate bounding range containing +all returned origin cells and the complete source ranges of merged origins. +It is not derived from the normalized grid dimensions. A range can contain +holes when hidden rows or columns were omitted. + +For a normal spreadsheet cell, `Cell.source` is a one-cell range. For a merged +origin, it is the complete source merge range, even if hidden rows or columns +cause the normalized span to be smaller. Covered grid slots continue to point +to their normalized origin; the origin cell carries the source range. + +Materialized empty padding cells also carry their original one-cell range. A +cell is allowed to have no source only when it was not produced by a +spreadsheet parser. + +## Proposed Rust model + +The following names are the proposed public API; exact naming can be adjusted +during upstream review without changing the semantics. + +```rust +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SpreadsheetCoordinate { + pub row: u32, + pub column: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SpreadsheetRange { + pub start: SpreadsheetCoordinate, + pub end: SpreadsheetCoordinate, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpreadsheetSource { + pub sheet_index: u32, + pub sheet_name: String, + pub range: SpreadsheetRange, +} + +pub struct Table { + pub grid: Vec>, + pub header_rows: usize, + pub kind: TableKind, + pub source: Option, +} + +pub struct Cell { + pub blocks: Vec, + pub col_span: u32, + pub row_span: u32, + pub source: Option, +} +``` + +`Table.source` is `Some` for tables produced by spreadsheet parsers and +`None` for DOC/DOCX, RTF, ODT, presentation, HTML-derived, and CSV tables. +The worksheet identity is stored once at table level; each cell stores only its +range to avoid duplicating the sheet name for every cell. + +## Parser changes + +### XLSX, XLSB, and XLS + +`xlsx`, `xlsb`, and `xls` already share `SheetContent` and `build_table()`. +The implementation should: + +1. carry a `SpreadsheetSource` sheet identity into `build_table()`; +2. retain the original `(row, column)` key while building each origin cell; +3. retain the original merge rectangle separately from its normalized span; +4. attach a one-cell range to generated empty cells; +5. compute the table bounding range from source coordinates before returning; +6. enumerate sheet identity before filtering hidden sheets. + +The normalized grid, merge behavior, number formatting, and hidden-content +policy must remain unchanged. + +### ODS + +`parse_spreadsheet()` and `parse_table()` need explicit logical source cursors +because ODS can encode a single cell or row with repeat attributes. + +- `number-rows-repeated` increments the source row for every emitted row; +- `number-columns-repeated` increments the source column for every emitted + cell; +- `number-columns-spanned` and `number-rows-spanned` produce an inclusive + source range covering the full origin span; +- `covered-table-cell` remains a covered slot when it belongs to an origin; +- a stray covered cell, if recovered as an empty origin, receives its source + coordinate; +- repeat expansion continues to use the existing safety budgets. + +## Binding contract + +All bindings expose the same data with their established naming conventions. + +| Binding | Public shape | +|---|---| +| Rust | `SpreadsheetCoordinate`, `SpreadsheetRange`, `SpreadsheetSource`; optional `Table.source` and `Cell.source` | +| Node.js | `sheetIndex`, `sheetName`, `range`, `start`, `end`, `row`, `column`; optional `source` | +| Python | `sheet_index`, `sheet_name`, `range`, `start`, `end`, `row`, `column`; optional `source` | +| WASM | camelCase JSON/TypeScript shape matching Node.js | + +The generated Node declaration, Python stub, and WASM TypeScript section must +be updated together with their conversion code. Non-spreadsheet output should +continue to expose `None`/`undefined`/omitted optional provenance according to +each binding's existing optional-field convention. + +## Test strategy + +### Core parser tests + +Add focused cases for each supported container: + +1. a single-sheet workbook with one value at `C3`; +2. multiple sheets with distinct names and a hidden sheet before a visible + sheet; +3. a non-`A1` used range such as `D11:E12`; +4. hidden rows and columns that change normalized grid positions; +5. merged cells whose source range differs from normalized span; +6. ODS repeated rows, repeated columns, and covered cells; +7. empty materialized padding cells; +8. non-spreadsheet and CSV tables with no provenance. + +The XLSX, XLSB, and XLS tests should exercise the shared grid path while still +asserting each format's reader. ODS tests should assert repeat and span cursor +behavior independently. + +### Regression tests + +- Existing Markdown snapshots and renderer tests remain unchanged. +- Existing table/grid invariants continue to pass. +- Binding tests assert the new fields and their absence for non-spreadsheet + tables. +- Existing malformed-input and resource-limit tests remain green. + +## Acceptance criteria + +The change is complete when: + +- `to_document()` returns `Data Sheet!C3:C3` and the cell range for a + single-value `C3` workbook; +- every returned spreadsheet origin cell is traceable to its original sheet + and inclusive source range; +- `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, and `.ods` follow the same coordinate + contract; +- normalized grid shape and Markdown output do not regress; +- all four public surfaces expose the new metadata consistently; +- tests cover hidden coordinates, merges, repeats, and empty padding; +- no CSR project files are modified by this upstream change. + +## Compatibility and review risks + +`Table` and `Cell` currently expose public fields, so adding public fields can +break downstream code that constructs them with struct literals. This is an +intentional public-model change required by the issue and must be called out +in the PR description and release notes. The optional values preserve runtime +behavior for non-spreadsheet documents. + +## Implementation order + +1. Add and document the shared provenance types and optional model fields. +2. Thread sheet identity and source ranges through the shared Excel grid path. +3. Add ODS logical source cursors and range tracking. +4. Update Rust, Node.js, Python, and WASM bindings. +5. Add parser, binding, and regression tests. +6. Review the complete diff, run the relevant test matrix, and prepare the + upstream PR description. diff --git a/node/index.d.ts b/node/index.d.ts index e3e4fd72..c3c9080b 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -81,6 +81,31 @@ export interface Cell { blocks: Array colSpan: number rowSpan: number + /** Inclusive source range for a spreadsheet origin cell. */ + source?: SpreadsheetRange +} + +export interface SpreadsheetCoordinate { + /** Zero-based row. */ + row: number + /** Zero-based column. */ + column: number +} + +export interface SpreadsheetRange { + /** Inclusive range start. */ + start: SpreadsheetCoordinate + /** Inclusive range end. */ + end: SpreadsheetCoordinate +} + +export interface SpreadsheetSource { + /** Zero-based position in the source workbook's worksheet order. */ + sheetIndex: number + /** Worksheet name as stored by the source format. */ + sheetName: string + /** Inclusive source extent that produced the returned table. */ + range: SpreadsheetRange } export interface CellSlot { @@ -275,6 +300,8 @@ export interface Table { /** Number of leading rows that are header rows (0 = no header). */ headerRows: number kind: TableKind + /** Worksheet identity and source extent for a spreadsheet table. */ + source?: SpreadsheetSource } export declare const enum TableKind { diff --git a/node/src/document.rs b/node/src/document.rs index 62f9b5d0..e6821c27 100644 --- a/node/src/document.rs +++ b/node/src/document.rs @@ -329,6 +329,53 @@ pub enum TableKind { layout, } +/// A zero-based row and column in a spreadsheet worksheet. +#[napi(object)] +pub struct SpreadsheetCoordinate { + pub row: u32, + pub column: u32, +} + +impl From for SpreadsheetCoordinate { + fn from(coordinate: model::SpreadsheetCoordinate) -> Self { + SpreadsheetCoordinate { row: coordinate.row, column: coordinate.column } + } +} + +/// An inclusive source range in a spreadsheet worksheet. +#[napi(object)] +pub struct SpreadsheetRange { + pub start: SpreadsheetCoordinate, + pub end: SpreadsheetCoordinate, +} + +impl From for SpreadsheetRange { + fn from(range: model::SpreadsheetRange) -> Self { + SpreadsheetRange { start: range.start.into(), end: range.end.into() } + } +} + +/// The worksheet and source extent of a returned spreadsheet table. +#[napi(object)] +pub struct SpreadsheetSource { + /// Zero-based position in the source workbook's worksheet order. + pub sheet_index: u32, + /// Worksheet name as stored by the source format. + pub sheet_name: String, + /// Inclusive source extent that produced the returned table. + pub range: SpreadsheetRange, +} + +impl From for SpreadsheetSource { + fn from(source: model::SpreadsheetSource) -> Self { + SpreadsheetSource { + sheet_index: source.sheet_index, + sheet_name: source.sheet_name, + range: source.range.into(), + } + } +} + /// Canonical table grid: every logical grid position appears exactly once. /// Content and spans live on the origin slot, and each position a span covers /// holds a `covered` slot pointing back at that origin. @@ -338,6 +385,8 @@ pub struct Table { /// Number of leading rows that are header rows (0 = no header). pub header_rows: u32, pub kind: TableKind, + /// Worksheet identity and source extent for a spreadsheet table. + pub source: Option, } impl From for Table { @@ -353,6 +402,7 @@ impl From for Table { model::TableKind::Data => TableKind::data, model::TableKind::Layout => TableKind::layout, }, + source: table.source.map(SpreadsheetSource::from), } } } @@ -399,11 +449,18 @@ pub struct Cell { pub blocks: Vec, pub col_span: u32, pub row_span: u32, + /// Inclusive source range for a spreadsheet origin cell. + pub source: Option, } impl From for Cell { fn from(cell: model::Cell) -> Self { - Cell { blocks: blocks(cell.blocks), col_span: cell.col_span, row_span: cell.row_span } + Cell { + blocks: blocks(cell.blocks), + col_span: cell.col_span, + row_span: cell.row_span, + source: cell.source.map(SpreadsheetRange::from), + } } } diff --git a/node/test.mjs b/node/test.mjs index 0d4f6a4b..b24af9e0 100644 --- a/node/test.mjs +++ b/node/test.mjs @@ -22,6 +22,7 @@ const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, imp const OUTLINE = fixture('docx/handmade-outline.docx') const RICH = fixture('docx/handmade-rich.docx') +const SPREADSHEET = fixture('xlsx/handmade-merged.xlsx') const CSV = fixture('csv/sheet.csv') const ENCRYPTED = fixture('malformed/encrypted--errors.odt') const MIXED = fixture('pdf/handmade-mixed.pdf') @@ -53,6 +54,31 @@ test('toDocument exposes the document model', async () => { assert.equal(typeof heading.content[0].style.bold, 'boolean') }) +test('toDocument exposes spreadsheet source coordinates', async () => { + const document = await toDocument(await readFile(SPREADSHEET), 'xlsx') + const table = document.blocks.find((block) => block.kind === 'table')?.table + assert.ok(table) + assert.equal(table.source.sheetIndex, 0) + assert.equal(table.source.sheetName, 'Merged') + assert.deepEqual(table.source.range, { + start: { row: 0, column: 0 }, + end: { row: 2, column: 2 }, + }) + assert.deepEqual(table.grid[0][0].cell.source, { + start: { row: 0, column: 0 }, + end: { row: 0, column: 1 }, + }) + assert.deepEqual(table.grid[1][0].cell.source, { + start: { row: 1, column: 0 }, + end: { row: 2, column: 0 }, + }) + + const docx = await toDocument(await readFile(RICH), 'docx') + const docxTable = docx.blocks.find((block) => block.kind === 'table')?.table + assert.ok(docxTable) + assert.equal(docxTable.source, undefined) +}) + test('toDocument carries embedded assets as buffers', async () => { const document = await toDocument(await readFile(RICH), 'docx') const image = document.assets.find((asset) => asset.mediaType === 'image/png') diff --git a/python/anydoc/__init__.py b/python/anydoc/__init__.py index c1ba6560..0965bec1 100644 --- a/python/anydoc/__init__.py +++ b/python/anydoc/__init__.py @@ -27,6 +27,9 @@ NeedsOcrError, Note, ResourceLimitError, + SpreadsheetCoordinate, + SpreadsheetRange, + SpreadsheetSource, Style, Table, UnsupportedError, @@ -201,6 +204,9 @@ def _version() -> str: "Note", "Ocr", "ResourceLimitError", + "SpreadsheetCoordinate", + "SpreadsheetRange", + "SpreadsheetSource", "Style", "Table", "UnsupportedError", diff --git a/python/anydoc/_anydoc.pyi b/python/anydoc/_anydoc.pyi index ca45a408..85d72bd4 100644 --- a/python/anydoc/_anydoc.pyi +++ b/python/anydoc/_anydoc.pyi @@ -173,6 +173,28 @@ class ListItem: number text cannot be reproduced from the marker and position alone (composite number text such as `1-a)`).""" +@final +class SpreadsheetCoordinate: + """A zero-based row and column in a spreadsheet worksheet.""" + + row: int + column: int + +@final +class SpreadsheetRange: + """An inclusive source range in a spreadsheet worksheet.""" + + start: SpreadsheetCoordinate + end: SpreadsheetCoordinate + +@final +class SpreadsheetSource: + """The worksheet and source extent of a returned spreadsheet table.""" + + sheet_index: int + sheet_name: str + range: SpreadsheetRange + @final class Table: """Canonical table grid: every logical grid position appears exactly @@ -185,6 +207,8 @@ class Table: kind: Literal["data", "layout"] """data: a real data table. layout: layout scaffolding (text boxes, positioning tables).""" + source: SpreadsheetSource | None + """Worksheet identity and source extent for a spreadsheet table.""" @final class CellSlot: @@ -201,6 +225,8 @@ class Cell: blocks: list[Block] col_span: int row_span: int + source: SpreadsheetRange | None + """Inclusive source range for a spreadsheet origin cell.""" @final class Note: diff --git a/python/src/document.rs b/python/src/document.rs index f0e2e844..168f3bea 100644 --- a/python/src/document.rs +++ b/python/src/document.rs @@ -258,6 +258,54 @@ fn list_item(py: Python<'_>, item: model::ListItem) -> PyResult { Ok(ListItem { blocks: blocks(py, item.blocks)?, marker_label: item.marker_label }) } +#[pyclass(frozen, get_all, module = "anydoc")] +pub struct SpreadsheetCoordinate { + /// Zero-based row. + row: usize, + /// Zero-based column. + column: usize, +} + +fn spreadsheet_coordinate(coordinate: model::SpreadsheetCoordinate) -> SpreadsheetCoordinate { + SpreadsheetCoordinate { row: coordinate.row as usize, column: coordinate.column as usize } +} + +#[pyclass(frozen, get_all, module = "anydoc")] +pub struct SpreadsheetRange { + /// Inclusive range start. + start: Py, + /// Inclusive range end. + end: Py, +} + +fn spreadsheet_range(py: Python<'_>, range: model::SpreadsheetRange) -> PyResult { + Ok(SpreadsheetRange { + start: Py::new(py, spreadsheet_coordinate(range.start))?, + end: Py::new(py, spreadsheet_coordinate(range.end))?, + }) +} + +#[pyclass(frozen, get_all, module = "anydoc")] +pub struct SpreadsheetSource { + /// Zero-based position in the source workbook's worksheet order. + sheet_index: usize, + /// Worksheet name as stored by the source format. + sheet_name: String, + /// Inclusive source extent that produced the returned table. + range: Py, +} + +fn spreadsheet_source( + py: Python<'_>, + source: model::SpreadsheetSource, +) -> PyResult { + Ok(SpreadsheetSource { + sheet_index: source.sheet_index as usize, + sheet_name: source.sheet_name, + range: Py::new(py, spreadsheet_range(py, source.range)?)?, + }) +} + /// Canonical table grid: every logical grid position appears exactly once. /// Content and spans live on the origin slot, and each position a span covers /// holds a `covered` slot pointing back at that origin. @@ -270,6 +318,8 @@ pub struct Table { /// data (a real data table) or layout (layout scaffolding: text boxes, /// positioning tables). kind: &'static str, + /// Worksheet identity and source extent for a spreadsheet table. + source: Option>, } fn table(py: Python<'_>, table: model::Table) -> PyResult { @@ -284,6 +334,10 @@ fn table(py: Python<'_>, table: model::Table) -> PyResult
{ model::TableKind::Data => "data", model::TableKind::Layout => "layout", }, + source: table + .source + .map(|source| spreadsheet_source(py, source).and_then(|source| Py::new(py, source))) + .transpose()?, }) } @@ -322,10 +376,20 @@ pub struct Cell { blocks: Py, col_span: u32, row_span: u32, + /// Inclusive source range for a spreadsheet origin cell. + source: Option>, } fn cell(py: Python<'_>, cell: model::Cell) -> PyResult { - Ok(Cell { blocks: blocks(py, cell.blocks)?, col_span: cell.col_span, row_span: cell.row_span }) + Ok(Cell { + blocks: blocks(py, cell.blocks)?, + col_span: cell.col_span, + row_span: cell.row_span, + source: cell + .source + .map(|source| spreadsheet_range(py, source).and_then(|source| Py::new(py, source))) + .transpose()?, + }) } #[pyclass(frozen, get_all, module = "anydoc")] diff --git a/python/src/lib.rs b/python/src/lib.rs index a38a09de..2f618361 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -215,6 +215,9 @@ fn _anydoc(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add("ConvertError", m.py().get_type::())?; diff --git a/python/tests/test_anydoc.py b/python/tests/test_anydoc.py index 8cff940e..66e7c659 100644 --- a/python/tests/test_anydoc.py +++ b/python/tests/test_anydoc.py @@ -16,6 +16,7 @@ FIXTURES = Path(__file__).resolve().parents[2] / "tests" / "fixtures" OUTLINE = FIXTURES / "docx" / "handmade-outline.docx" RICH = FIXTURES / "docx" / "handmade-rich.docx" +SPREADSHEET = FIXTURES / "xlsx" / "handmade-merged.xlsx" CSV = FIXTURES / "csv" / "sheet.csv" ENCRYPTED = FIXTURES / "malformed" / "encrypted--errors.odt" ZIPBOMB = FIXTURES / "abuse" / "zipbomb--errors.docx" @@ -86,6 +87,36 @@ def test_to_document_exposes_the_document_model(self): self.assertEqual(heading.content[0].kind, "text") self.assertIsInstance(heading.content[0].style.bold, bool) + def test_to_document_exposes_spreadsheet_source_coordinates(self): + document = anydoc.to_document(SPREADSHEET.read_bytes(), "xlsx") + table = next((block.table for block in document.blocks if block.kind == "table"), None) + self.assertIsNotNone(table, "spreadsheet fixture did not produce a table") + self.assertEqual(table.source.sheet_index, 0) + self.assertEqual(table.source.sheet_name, "Merged") + self.assertEqual((table.source.range.start.row, table.source.range.start.column), (0, 0)) + self.assertEqual((table.source.range.end.row, table.source.range.end.column), (2, 2)) + self.assertEqual( + (table.grid[0][0].cell.source.start.row, table.grid[0][0].cell.source.start.column), + (0, 0), + ) + self.assertEqual( + (table.grid[0][0].cell.source.end.row, table.grid[0][0].cell.source.end.column), + (0, 1), + ) + self.assertEqual( + (table.grid[1][0].cell.source.start.row, table.grid[1][0].cell.source.start.column), + (1, 0), + ) + self.assertEqual( + (table.grid[1][0].cell.source.end.row, table.grid[1][0].cell.source.end.column), + (2, 0), + ) + + docx = anydoc.to_document(RICH.read_bytes(), "docx") + docx_table = next((block.table for block in docx.blocks if block.kind == "table"), None) + self.assertIsNotNone(docx_table, "DOCX fixture did not produce a table") + self.assertIsNone(docx_table.source) + def test_to_document_carries_embedded_assets_as_bytes(self): document = anydoc.to_document(RICH.read_bytes(), "docx") image = next(asset for asset in document.assets if asset.media_type == "image/png") diff --git a/src/formats/csv.rs b/src/formats/csv.rs index 5d4618b8..71891e55 100644 --- a/src/formats/csv.rs +++ b/src/formats/csv.rs @@ -118,7 +118,9 @@ mod tests { fn quoted_fields_keep_padding() { let doc = parse(b"a,b\n\" padded \",x\n").unwrap(); let Block::Table(t) = &doc.blocks[0] else { panic!() }; + assert!(t.source.is_none()); let crate::model::CellSlot::Origin(cell) = &t.grid[1][0] else { panic!() }; + assert!(cell.source.is_none()); let Block::Paragraph(inlines) = &cell.blocks[0] else { panic!() }; let crate::model::Inline::Text { text, .. } = &inlines[0] else { panic!() }; assert_eq!(text, " padded "); diff --git a/src/formats/odf/mod.rs b/src/formats/odf/mod.rs index e4a64553..0d68a215 100644 --- a/src/formats/odf/mod.rs +++ b/src/formats/odf/mod.rs @@ -177,6 +177,7 @@ fn push_title_heading(inner: Vec, blocks: &mut Vec) { #[cfg(test)] mod tests { use super::*; + use crate::model::{CellSlot, SpreadsheetRange, Table}; use std::io::{Cursor, Write}; const CONTENT: &[u8] = br#" String { + format!( + r#" + {tables} + "# + ) + } + #[test] fn repeated_rows_cannot_amplify_text_beyond_the_byte_budget() { // H3: the slot budget alone would admit 1000 copies of a 100 KB @@ -281,6 +293,105 @@ mod tests { panic!("unexpected cell blocks: {:?}", cell.blocks) }; assert_eq!(inside, "inside cell"); + assert!(table.source.is_none(), "ODT tables do not have worksheet provenance"); + } + + #[test] + fn spreadsheet_sources_follow_repeat_and_span_cursors() { + let content = spreadsheet_doc( + r#" + + + + + + + + + + + + + + + + + + + "#, + ); + let doc = parse(&odt_with_content(&content)).unwrap(); + let tables: Vec<&Table> = doc + .blocks + .iter() + .filter_map(|block| match block { + Block::Table(table) => Some(table), + _ => None, + }) + .collect(); + assert_eq!(tables.len(), 2); + + let first_source = tables[0].source.as_ref().expect("spreadsheet source"); + assert_eq!(first_source.sheet_index, 0); + assert_eq!(first_source.sheet_name, "Data Sheet"); + assert_eq!(first_source.range, SpreadsheetRange::new(0, 0, 2, 4)); + let CellSlot::Origin(value) = &tables[0].grid[0][2] else { + panic!("expected the value origin at C1"); + }; + assert_eq!(value.source, Some(SpreadsheetRange::cell(0, 2))); + let CellSlot::Origin(repeated_one) = &tables[0].grid[1][0] else { + panic!("expected the first repeated row"); + }; + let CellSlot::Origin(repeated_two) = &tables[0].grid[2][0] else { + panic!("expected the second repeated row"); + }; + assert_eq!(repeated_one.source, Some(SpreadsheetRange::cell(1, 0))); + assert_eq!(repeated_two.source, Some(SpreadsheetRange::cell(2, 0))); + + let second_source = tables[1].source.as_ref().expect("spreadsheet source"); + assert_eq!(second_source.sheet_index, 1); + assert_eq!(second_source.sheet_name, "Merged"); + assert_eq!(second_source.range, SpreadsheetRange::new(0, 0, 1, 2)); + let CellSlot::Origin(span) = &tables[1].grid[0][0] else { + panic!("expected the merged origin"); + }; + assert_eq!((span.col_span, span.row_span), (2, 2)); + assert_eq!(span.source, Some(SpreadsheetRange::new(0, 0, 1, 1))); + assert!(matches!(tables[1].grid[0][1], CellSlot::Covered { .. })); + assert!(matches!(tables[1].grid[1][0], CellSlot::Covered { .. })); + assert!(matches!(tables[1].grid[1][1], CellSlot::Covered { .. })); + let CellSlot::Origin(tail) = &tables[1].grid[1][2] else { + panic!("expected the tail origin"); + }; + assert_eq!(tail.source, Some(SpreadsheetRange::cell(1, 2))); + } + + #[test] + fn spreadsheet_merge_source_survives_a_covered_tail() { + let content = spreadsheet_doc( + r#" + + + + + + "#, + ); + let doc = parse(&odt_with_content(&content)).unwrap(); + let Block::Table(table) = &doc.blocks[0] else { + panic!("expected one spreadsheet table"); + }; + let source = table.source.as_ref().expect("spreadsheet source"); + assert_eq!(source.sheet_name, "Trailing Merge"); + assert_eq!(source.range, SpreadsheetRange::new(0, 0, 2, 0)); + assert_eq!(table.grid.len(), 3); + let CellSlot::Origin(origin) = &table.grid[0][0] else { + panic!("expected the merged origin"); + }; + assert_eq!(origin.row_span, 3); + assert_eq!(origin.source, Some(SpreadsheetRange::new(0, 0, 2, 0))); + assert!(matches!(table.grid[1][0], CellSlot::Covered { .. })); + assert!(matches!(table.grid[2][0], CellSlot::Covered { .. })); } #[test] diff --git a/src/formats/odf/table.rs b/src/formats/odf/table.rs index 226446b2..15f95271 100644 --- a/src/formats/odf/table.rs +++ b/src/formats/odf/table.rs @@ -10,7 +10,10 @@ use crate::error::ConvertError; use crate::formats::odf::text::{Ctx, parse_container}; -use crate::model::{Block, Cell, GridBuilder, Inline, TableKind}; +use crate::model::{ + Block, Cell, CellSlot, GridBuilder, Inline, SpreadsheetRange, SpreadsheetSource, Table, + TableKind, +}; use crate::package::limits; use crate::package::xml::{Element, ns}; use crate::shared::header::resolve_header_rows; @@ -18,6 +21,14 @@ use crate::shared::text::clean_text; use std::collections::HashMap; pub fn parse_table(elem: &Element, ctx: &Ctx) -> Result, ConvertError> { + parse_table_with_source(elem, ctx, None) +} + +fn parse_table_with_source( + elem: &Element, + ctx: &Ctx, + source: Option<(u32, &str)>, +) -> Result, ConvertError> { let mut state = TableState { builder: GridBuilder::new(), expansion: 0, @@ -25,14 +36,27 @@ pub fn parse_table(elem: &Element, ctx: &Ctx) -> Result, ConvertError pending_rows: 0, header_rows: 0, rows_emitted: 0, + next_source_row: 0, checkboxes: read_checkboxes(elem), }; + // A spreadsheet merge is real source extent even when its final rows + // contain only covered cells. Ordinary ODF tables keep the historical + // trailing-covered-row trim, so enable this only for spreadsheet tables. + if source.is_some() { + state.builder.keep_covered_tail(); + } walk_rows(elem, ctx, &mut state, true)?; let mut table = state.builder.finish(TableKind::Data); if table.grid.is_empty() { return Ok(Vec::new()); } table.header_rows = resolve_header_rows(&table, state.header_rows); + if let Some((sheet_index, sheet_name)) = source + && let Some(range) = table_source_range(&table) + { + table.source = + Some(SpreadsheetSource { sheet_index, sheet_name: sheet_name.to_string(), range }); + } Ok(vec![Block::Table(table)]) } @@ -45,6 +69,8 @@ struct TableState { pending_rows: u64, header_rows: usize, rows_emitted: usize, + /// Source row of the next row element, including rows buffered as empty. + next_source_row: u64, /// The sheet's form checkboxes by control id, as the inlines a /// `draw:control` in a cell expands to. checkboxes: HashMap>, @@ -217,6 +243,8 @@ fn emit_row( state: &mut TableState, repeat: u64, ) -> Result<(), ConvertError> { + let source_row = state.next_source_row; + state.next_source_row = state.next_source_row.saturating_add(repeat); if row_is_empty(row) { state.pending_rows = state.pending_rows.saturating_add(repeat); return Ok(()); @@ -243,10 +271,11 @@ fn emit_row( state.charge_bytes(bytes.saturating_mul(copies))?; } } - for _ in 0..repeat { + for row_offset in 0..repeat { + let emitted_source_row = source_row.saturating_add(row_offset); state.builder.next_row(); state.rows_emitted += 1; - emit_parsed_cells(&cells, state)?; + emit_parsed_cells(&cells, state, emitted_source_row)?; } Ok(()) } @@ -289,31 +318,48 @@ fn parse_row_cells( Ok(out) } -fn emit_parsed_cells(cells: &[RowCell], state: &mut TableState) -> Result<(), ConvertError> { +fn emit_parsed_cells( + cells: &[RowCell], + state: &mut TableState, + source_row: u64, +) -> Result<(), ConvertError> { let mut pending_cells: u64 = 0; + let mut source_col: u64 = 0; for cell in cells { match cell { RowCell::Covered { repeat } => { - flush_gap(state, &mut pending_cells)?; + let gap_start = source_col.saturating_sub(pending_cells); + flush_gap(state, &mut pending_cells, source_row, gap_start)?; // One explicitly written covered position each; a stray one // (no span accounts for it) becomes an empty cell inside // covered(). state.charge(*repeat)?; for _ in 0..*repeat { - if !state.builder.covered() { + let source = source_cell(source_row, source_col)?; + if !state.builder.covered_with(Cell { source: Some(source), ..Cell::default() }) + { log::debug!("covered table cell without a spanning origin"); } + source_col = source_col.saturating_add(1); } } RowCell::Cell { repeat, col_span, row_span, blocks, .. } => { if blocks.is_empty() && *col_span == 1 && *row_span == 1 { pending_cells = pending_cells.saturating_add(*repeat); + source_col = source_col.saturating_add(*repeat); continue; } - flush_gap(state, &mut pending_cells)?; + let gap_start = source_col.saturating_sub(pending_cells); + flush_gap(state, &mut pending_cells, source_row, gap_start)?; state.charge(repeat.saturating_mul(*col_span as u64))?; for _ in 0..*repeat { - state.builder.place(Cell::spanning(blocks.clone(), *col_span, *row_span))?; + let end_row = source_row.saturating_add(u64::from(*row_span).saturating_sub(1)); + let end_col = source_col.saturating_add(u64::from(*col_span).saturating_sub(1)); + let source = source_range(source_row, source_col, end_row, end_col)?; + let mut origin = Cell::spanning(blocks.clone(), *col_span, *row_span); + origin.source = Some(source); + state.builder.place(origin)?; + source_col = source_col.saturating_add(u64::from(*col_span)); } } } @@ -323,18 +369,67 @@ fn emit_parsed_cells(cells: &[RowCell], state: &mut TableState) -> Result<(), Co /// Materialize a buffered empty-cell run in full so the next cell lands on /// its source column. Trailing runs are never flushed and stay elided. -fn flush_gap(state: &mut TableState, pending: &mut u64) -> Result<(), ConvertError> { +fn flush_gap( + state: &mut TableState, + pending: &mut u64, + source_row: u64, + source_col: u64, +) -> Result<(), ConvertError> { if *pending == 0 { return Ok(()); } state.charge(*pending)?; - for _ in 0..*pending { - state.builder.place(Cell::default())?; + for offset in 0..*pending { + let source = source_cell(source_row, source_col.saturating_add(offset))?; + state.builder.place(Cell { source: Some(source), ..Cell::default() })?; } *pending = 0; Ok(()) } +fn source_cell(row: u64, column: u64) -> Result { + source_range(row, column, row, column) +} + +fn source_range( + start_row: u64, + start_column: u64, + end_row: u64, + end_column: u64, +) -> Result { + let to_u32 = |value: u64, axis: &str| { + u32::try_from(value).map_err(|_| ConvertError::ResourceLimit { + limit: "spreadsheet_coordinates", + detail: format!("{axis} coordinate {value} exceeds the model range"), + }) + }; + Ok(SpreadsheetRange::new( + to_u32(start_row, "row")?, + to_u32(start_column, "column")?, + to_u32(end_row, "row")?, + to_u32(end_column, "column")?, + )) +} + +fn table_source_range(table: &Table) -> Option { + let mut range: Option = None; + for row in &table.grid { + for slot in row { + let CellSlot::Origin(cell) = slot else { + continue; + }; + let Some(source) = cell.source else { + continue; + }; + match &mut range { + Some(found) => found.include(source), + None => range = Some(source), + } + } + } + range +} + /// A cell's blocks: its text content, or a typed value-attribute fallback /// when the producer wrote no display text. fn cell_blocks(cell: &Element, ctx: &Ctx) -> Result, ConvertError> { @@ -463,9 +558,9 @@ pub fn parse_spreadsheet(sheet: &Element, ctx: &Ctx) -> Result, Conve let tables: Vec<&Element> = sheet.child_elems().filter(|e| e.is(ns::TABLE, "table")).collect(); let multi_sheet = tables.len() > 1; let mut blocks = Vec::new(); - for table in tables { + for (sheet_index, table) in tables.into_iter().enumerate() { let name = table.attr(ns::TABLE, "name").unwrap_or(""); - let content = parse_table(table, ctx)?; + let content = parse_table_with_source(table, ctx, Some((sheet_index as u32, name)))?; if content.is_empty() { continue; } diff --git a/src/formats/sheet/xls.rs b/src/formats/sheet/xls.rs index 28488fdc..0cd13919 100644 --- a/src/formats/sheet/xls.rs +++ b/src/formats/sheet/xls.rs @@ -59,19 +59,25 @@ pub(super) fn parse(bytes: &[u8]) -> Result { let mut records = 0u64; let globals = read_globals(&data, &mut records)?; - let visible: Vec<&BoundSheet> = globals.sheets.iter().filter(|s| s.visible).collect(); + let visible: Vec<(u32, &BoundSheet)> = globals + .sheets + .iter() + .enumerate() + .filter(|(_, s)| s.visible) + .map(|(index, sheet)| (index as u32, sheet)) + .collect(); let multi_sheet = visible.len() > 1; let mut doc = Document::default(); let mut failed = 0usize; // One budget for the workbook, so sheets cannot multiply the cap. let mut slots = 0u64; - for sheet in &visible { + for (sheet_index, sheet) in &visible { let Some(content) = read_sheet(&data, &globals, sheet.offset, &mut records)? else { log::warn!("skipping unreadable sheet {:?}", sheet.name); failed += 1; continue; }; - let Some(table) = build_table(content, &mut slots)? else { + let Some(table) = build_table(content, *sheet_index, &sheet.name, &mut slots)? else { continue; }; if multi_sheet { @@ -840,7 +846,7 @@ fn string_reader<'a>(segs: &'a [&'a [u8]], skip: usize) -> Option> #[cfg(test)] mod tests { use super::*; - use crate::model::{CellSlot, Table, inlines_to_plain_text}; + use crate::model::{CellSlot, SpreadsheetRange, Table, inlines_to_plain_text}; use std::io::Write; fn rec(rec_type: u16, body: &[u8]) -> Vec { @@ -1140,7 +1146,65 @@ mod tests { }; let doc = parse(&wb.build()).unwrap(); assert_eq!(doc.blocks.len(), 1, "hidden sheet must add no heading and no table"); - assert_eq!(texts(first_table(&doc)), vec![vec!["a", "c"], vec!["d", ""]]); + let table = first_table(&doc); + assert_eq!(texts(table), vec![vec!["a", "c"], vec!["d", ""]]); + let CellSlot::Origin(cell) = &table.grid[0][0] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(0, 0))); + let CellSlot::Origin(cell) = &table.grid[0][1] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(0, 2))); + let CellSlot::Origin(cell) = &table.grid[1][0] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(2, 0))); + let CellSlot::Origin(cell) = &table.grid[1][1] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(2, 2))); + } + + #[test] + fn source_coordinates_keep_workbook_order_and_sparse_extents() { + let first = label(2, 2, 0, "value"); + let hidden = label(0, 0, 0, "hidden"); + let mut third = label(10, 3, 0, "d"); + third.extend(label(11, 4, 0, "e")); + let wb = Wb { + xfs: vec![0], + sheets: vec![("Data Sheet", 0, first), ("Hidden", 1, hidden), ("Report", 0, third)], + ..Wb::default() + }; + let doc = parse(&wb.build()).unwrap(); + let tables: Vec<&Table> = doc + .blocks + .iter() + .filter_map(|block| match block { + Block::Table(table) => Some(table), + _ => None, + }) + .collect(); + assert_eq!(tables.len(), 2); + + let first_source = tables[0].source.as_ref().expect("spreadsheet source"); + assert_eq!(first_source.sheet_index, 0); + assert_eq!(first_source.sheet_name, "Data Sheet"); + assert_eq!(first_source.range, SpreadsheetRange::cell(2, 2)); + let CellSlot::Origin(cell) = &tables[0].grid[0][0] else { + panic!("expected C3 origin"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(2, 2))); + + let third_source = tables[1].source.as_ref().expect("spreadsheet source"); + assert_eq!(third_source.sheet_index, 2, "hidden sheet must not renumber provenance"); + assert_eq!(third_source.sheet_name, "Report"); + assert_eq!(third_source.range, SpreadsheetRange::new(10, 3, 11, 4)); + let CellSlot::Origin(cell) = &tables[1].grid[0][0] else { + panic!("expected D11 origin"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(10, 3))); + let CellSlot::Origin(cell) = &tables[1].grid[0][1] else { + panic!("expected generated E11 padding"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(10, 4))); + let CellSlot::Origin(cell) = &tables[1].grid[1][1] else { + panic!("expected E12 origin"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(11, 4))); } #[test] @@ -1161,6 +1225,7 @@ mod tests { panic!("expected the merge origin at (0,0)"); }; assert_eq!((cell.col_span, cell.row_span), (10, 3)); + assert_eq!(cell.source, Some(SpreadsheetRange::new(0, 5, 2, 14))); } #[test] diff --git a/src/formats/sheet/xlsb.rs b/src/formats/sheet/xlsb.rs index 7bc042f0..4713257a 100644 --- a/src/formats/sheet/xlsb.rs +++ b/src/formats/sheet/xlsb.rs @@ -59,15 +59,17 @@ pub(super) fn parse(pkg: &mut Package, wb_part: &str) -> Result = Vec::new(); - for (name, rid) in bundles { + // in xlsx: by relationship id, never by conventional part name. Each + // tuple retains the original workbook-order index before hidden sheets + // are omitted. + let mut sheets: Vec<(u32, String, String)> = Vec::new(); + for (sheet_index, name, rid) in bundles { let Some(target) = wb_rels.internal_target(&rid) else { log::warn!("skipping sheet {name:?} with no worksheet relationship"); continue; }; match path::resolve(wb_part, target) { - Ok(t) => sheets.push((name, t.path)), + Ok(t) => sheets.push((sheet_index, name, t.path)), Err(e) => log::warn!("skipping sheet {name:?} with unresolvable target: {e}"), } } @@ -77,7 +79,7 @@ pub(super) fn parse(pkg: &mut Package, wb_part: &str) -> Result Result( } } +type SheetInfo = (u32, String, String); + /// `xl/workbook.bin`: the 1904 date flag from BrtWbProp, and each visible -/// sheet's name and relationship id from its BrtBundleSh. -fn read_workbook(data: &[u8]) -> Result<(bool, Vec<(String, String)>), ConvertError> { +/// sheet's source-order index, name, and relationship id from its BrtBundleSh. +fn read_workbook(data: &[u8]) -> Result<(bool, Vec), ConvertError> { let mut date1904 = false; let mut sheets = Vec::new(); + let mut sheet_index = 0u32; let mut records = Records::new(data); while let Some((id, payload)) = records.next()? { match id { @@ -145,13 +150,15 @@ fn read_workbook(data: &[u8]) -> Result<(bool, Vec<(String, String)>), ConvertEr f.u32()?; // iTabID let rid = f.nullable_wide_string()?; let name = clean_text(&f.wide_string()?); + let current_index = sheet_index; + sheet_index = sheet_index.saturating_add(1); // hsState 1 is hidden, 2 is veryHidden: omitted entirely, // heading included, exactly as in xlsx. if state == 1 || state == 2 { continue; } match rid { - Some(rid) => sheets.push((name, rid)), + Some(rid) => sheets.push((current_index, name, rid)), None => log::warn!("skipping sheet {name:?} with no worksheet relationship"), } } @@ -460,7 +467,7 @@ impl<'a> Fields<'a> { #[cfg(test)] mod tests { use super::*; - use crate::model::{CellSlot, Table, inlines_to_plain_text}; + use crate::model::{CellSlot, SpreadsheetRange, Table, inlines_to_plain_text}; use std::io::Write; const PKG_RELS: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; @@ -822,7 +829,71 @@ mod tests { }; let doc = parse(&wb.build()).unwrap(); assert_eq!(doc.blocks.len(), 1, "hidden sheets must add no heading and no table"); - assert_eq!(texts(first_table(&doc)), vec![vec!["1", "3"], vec!["5", ""]]); + let table = first_table(&doc); + assert_eq!(texts(table), vec![vec!["1", "3"], vec!["5", ""]]); + let CellSlot::Origin(cell) = &table.grid[0][0] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(0, 0))); + let CellSlot::Origin(cell) = &table.grid[0][1] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(0, 2))); + let CellSlot::Origin(cell) = &table.grid[1][0] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(2, 0))); + let CellSlot::Origin(cell) = &table.grid[1][1] else { panic!() }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(2, 2))); + } + + #[test] + fn source_coordinates_keep_workbook_order_and_sparse_extents() { + let mut first = row_hdr(2, false); + first.extend(real_cell(2, 0, 7.0)); + let hidden = { + let mut body = row_hdr(0, false); + body.extend(real_cell(0, 0, 8.0)); + body + }; + let mut third = row_hdr(10, false); + third.extend(real_cell(3, 0, 1.0)); + third.extend(row_hdr(11, false)); + third.extend(real_cell(4, 0, 2.0)); + let wb = Wb { + sheets: vec![("Data Sheet", 0, first), ("Hidden", 1, hidden), ("Report", 0, third)], + ..Wb::default() + }; + let doc = parse(&wb.build()).unwrap(); + let tables: Vec<&Table> = doc + .blocks + .iter() + .filter_map(|block| match block { + Block::Table(table) => Some(table), + _ => None, + }) + .collect(); + assert_eq!(tables.len(), 2); + + let first_source = tables[0].source.as_ref().expect("spreadsheet source"); + assert_eq!(first_source.sheet_index, 0); + assert_eq!(first_source.sheet_name, "Data Sheet"); + assert_eq!(first_source.range, SpreadsheetRange::cell(2, 2)); + let CellSlot::Origin(cell) = &tables[0].grid[0][0] else { + panic!("expected C3 origin"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(2, 2))); + + let third_source = tables[1].source.as_ref().expect("spreadsheet source"); + assert_eq!(third_source.sheet_index, 2, "hidden sheet must not renumber provenance"); + assert_eq!(third_source.sheet_name, "Report"); + assert_eq!(third_source.range, SpreadsheetRange::new(10, 3, 11, 4)); + let CellSlot::Origin(cell) = &tables[1].grid[0][0] else { + panic!("expected D11 origin"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(10, 3))); + let CellSlot::Origin(cell) = &tables[1].grid[0][1] else { + panic!("expected generated E11 padding"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(10, 4))); + let CellSlot::Origin(cell) = &tables[1].grid[1][1] else { + panic!("expected E12 origin"); + }; + assert_eq!(cell.source, Some(SpreadsheetRange::cell(11, 4))); } #[test] @@ -843,6 +914,7 @@ mod tests { panic!("expected the merge origin at (0,0)"); }; assert_eq!((cell.col_span, cell.row_span), (10, 3)); + assert_eq!(cell.source, Some(SpreadsheetRange::new(0, 5, 2, 14))); } #[test] diff --git a/src/formats/sheet/xlsx.rs b/src/formats/sheet/xlsx.rs index e0924562..d989cab6 100644 --- a/src/formats/sheet/xlsx.rs +++ b/src/formats/sheet/xlsx.rs @@ -7,7 +7,10 @@ use super::controls::{Checkboxes, cell_inlines, read_vml_checkboxes}; use super::numfmt::{DateParts, NumberFormat, Rendered, builtin_code}; use super::{format_duration_days, format_float, format_time_of_day}; use crate::error::ConvertError; -use crate::model::{Block, Cell, Document, GridBuilder, Inline, Table, TableKind}; +use crate::model::{ + Block, Cell, Document, GridBuilder, Inline, SpreadsheetRange, SpreadsheetSource, Table, + TableKind, +}; use crate::package::limits; use crate::package::relationships::{Relationships, read_rels, rel_type, rels_part_for}; use crate::package::xml::{Element, ns}; @@ -48,11 +51,12 @@ pub(super) fn parse(pkg: &mut Package, wb_part: &str) -> Resultmerged"#, + ); + let doc = parse(&wb.build()).unwrap(); + let table = first_table(&doc); + assert_eq!(table.grid.len(), 1); + assert_eq!(table.grid[0].len(), 1); + assert_eq!( + table.source.as_ref().expect("spreadsheet source").range, + SpreadsheetRange::new(0, 0, 1, 1) + ); + let CellSlot::Origin(cell) = &table.grid[0][0] else { + panic!("expected the collapsed merge origin"); + }; + assert_eq!((cell.col_span, cell.row_span), (1, 1)); + assert_eq!(cell.source, Some(SpreadsheetRange::new(0, 0, 1, 1))); + } + #[test] fn merge_origin_in_a_hidden_row_keeps_its_content() { // The origin row is hidden but the merge survives: its value moves @@ -984,6 +1099,7 @@ mod tests { panic!("expected the merge origin at (0,0)"); }; assert_eq!(cell.row_span, 2); + assert_eq!(cell.source, Some(SpreadsheetRange::new(0, 0, 2, 0))); assert_eq!(texts(table)[0][0], "kept"); } diff --git a/src/model/mod.rs b/src/model/mod.rs index e5e835a7..a628a58a 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -10,6 +10,7 @@ mod block; mod inline; mod link; mod list; +mod source; mod style; mod table; @@ -18,6 +19,7 @@ pub use block::Block; pub use inline::{Inline, checkbox_text, inlines_are_empty, inlines_to_plain_text}; pub use link::{AnchorId, ImageSource, LinkTarget}; pub use list::{List, ListItem, MarkerKind}; +pub use source::{SpreadsheetCoordinate, SpreadsheetRange, SpreadsheetSource}; pub use style::Style; pub use table::{Cell, CellSlot, Table, TableKind}; diff --git a/src/model/source.rs b/src/model/source.rs new file mode 100644 index 00000000..b263adf3 --- /dev/null +++ b/src/model/source.rs @@ -0,0 +1,73 @@ +//! Source coordinates retained by format frontends. + +/// A zero-based row and column in a spreadsheet worksheet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SpreadsheetCoordinate { + /// Zero-based row. + pub row: u32, + /// Zero-based column. + pub column: u32, +} + +impl SpreadsheetCoordinate { + /// Construct a zero-based worksheet coordinate. + pub const fn new(row: u32, column: u32) -> Self { + SpreadsheetCoordinate { row, column } + } +} + +/// An inclusive source range in a spreadsheet worksheet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SpreadsheetRange { + /// Inclusive range start. + pub start: SpreadsheetCoordinate, + /// Inclusive range end. + pub end: SpreadsheetCoordinate, +} + +impl SpreadsheetRange { + /// Construct a range containing one worksheet cell. + pub const fn cell(row: u32, column: u32) -> Self { + let coordinate = SpreadsheetCoordinate::new(row, column); + SpreadsheetRange { start: coordinate, end: coordinate } + } + + /// Construct an inclusive range from its four zero-based bounds. + pub const fn new(start_row: u32, start_column: u32, end_row: u32, end_column: u32) -> Self { + SpreadsheetRange { + start: SpreadsheetCoordinate::new(start_row, start_column), + end: SpreadsheetCoordinate::new(end_row, end_column), + } + } + + /// Expand this range to include another inclusive range. + pub fn include(&mut self, other: SpreadsheetRange) { + self.start.row = self.start.row.min(other.start.row); + self.start.column = self.start.column.min(other.start.column); + self.end.row = self.end.row.max(other.end.row); + self.end.column = self.end.column.max(other.end.column); + } +} + +/// The worksheet and source extent of a returned spreadsheet table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpreadsheetSource { + /// Zero-based position in the source workbook's worksheet order. + pub sheet_index: u32, + /// Worksheet name as stored by the source format. + pub sheet_name: String, + /// Inclusive source extent that produced the returned table. + pub range: SpreadsheetRange, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ranges_are_inclusive_and_can_form_a_bounding_box() { + let mut range = SpreadsheetRange::cell(2, 4); + range.include(SpreadsheetRange::new(0, 1, 5, 3)); + assert_eq!(range, SpreadsheetRange::new(0, 1, 5, 4)); + } +} diff --git a/src/model/table.rs b/src/model/table.rs index 94941580..53665cf7 100644 --- a/src/model/table.rs +++ b/src/model/table.rs @@ -1,5 +1,5 @@ use crate::error::ConvertError; -use crate::model::{Block, Inline, inlines_are_empty}; +use crate::model::{Block, Inline, SpreadsheetRange, SpreadsheetSource, inlines_are_empty}; use crate::package::limits; use std::collections::HashMap; @@ -16,6 +16,10 @@ pub struct Table { pub header_rows: usize, /// Whether the source used this table for data or for layout. pub kind: TableKind, + /// Worksheet identity and source extent for a spreadsheet table. + /// + /// This is `None` for tables from formats without worksheet coordinates. + pub source: Option, } /// What a table is for. @@ -53,12 +57,17 @@ pub struct Cell { pub col_span: u32, /// Rows covered, at least 1. pub row_span: u32, + /// Inclusive source range for a spreadsheet origin cell. + /// + /// A normal cell has a one-cell range; a merged origin has the complete + /// source merge range. This is `None` for cells from other formats. + pub source: Option, } impl Cell { /// A cell spanning one position. pub fn new(blocks: Vec) -> Self { - Cell { blocks, col_span: 1, row_span: 1 } + Cell { blocks, col_span: 1, row_span: 1, source: None } } /// A one-paragraph cell spanning one position. @@ -69,7 +78,7 @@ impl Cell { /// A cell covering `col_span` by `row_span` positions; either span given /// as 0 is raised to 1. pub fn spanning(blocks: Vec, col_span: u32, row_span: u32) -> Self { - Cell { blocks, col_span: col_span.max(1), row_span: row_span.max(1) } + Cell { blocks, col_span: col_span.max(1), row_span: row_span.max(1), source: None } } /// True when the cell holds nothing that would render: only paragraphs @@ -217,6 +226,12 @@ impl GridBuilder { /// `covered-table-cell`). Returns `false` when no span accounts for the /// position - the stray marker then becomes an empty cell. pub fn covered(&mut self) -> bool { + self.covered_with(Cell::default()) + } + + /// Consume one explicitly-written covered position, using `fallback` when + /// no span accounts for the position. + pub fn covered_with(&mut self, fallback: Cell) -> bool { let row = self.row_index(); let col = self.grid[row].len(); match self.pending.remove(&(row, col)) { @@ -225,7 +240,7 @@ impl GridBuilder { true } None => { - self.grid[row].push(CellSlot::Origin(Cell::default())); + self.grid[row].push(CellSlot::Origin(fallback)); false } } @@ -276,7 +291,7 @@ impl GridBuilder { } } } - Table { grid: self.grid, header_rows: 0, kind } + Table { grid: self.grid, header_rows: 0, kind, source: None } } } diff --git a/src/render/markdown/tests.rs b/src/render/markdown/tests.rs index 0a7b19a4..53c999e3 100644 --- a/src/render/markdown/tests.rs +++ b/src/render/markdown/tests.rs @@ -50,7 +50,12 @@ fn math_renders_in_dollar_delimiters_and_text_dollars_are_escaped() { #[test] fn math_in_a_table_cell_escapes_pipes() { - let cell = |inlines| Cell { blocks: vec![Block::Paragraph(inlines)], col_span: 1, row_span: 1 }; + let cell = |inlines| Cell { + blocks: vec![Block::Paragraph(inlines)], + col_span: 1, + row_span: 1, + source: None, + }; let md = doc(vec![table_from( vec![vec![cell(vec![Inline::plain("abs")]), cell(vec![Inline::Math("|x|".into())])]], 0, diff --git a/wasm/src/document.rs b/wasm/src/document.rs index 287ae6d4..0ffab80b 100644 --- a/wasm/src/document.rs +++ b/wasm/src/document.rs @@ -352,6 +352,57 @@ pub enum TableKind { Layout, } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SpreadsheetCoordinate { + /// Zero-based row. + pub row: u32, + /// Zero-based column. + pub column: u32, +} + +impl From for SpreadsheetCoordinate { + fn from(coordinate: model::SpreadsheetCoordinate) -> Self { + SpreadsheetCoordinate { row: coordinate.row, column: coordinate.column } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SpreadsheetRange { + /// Inclusive range start. + pub start: SpreadsheetCoordinate, + /// Inclusive range end. + pub end: SpreadsheetCoordinate, +} + +impl From for SpreadsheetRange { + fn from(range: model::SpreadsheetRange) -> Self { + SpreadsheetRange { start: range.start.into(), end: range.end.into() } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SpreadsheetSource { + /// Zero-based position in the source workbook's worksheet order. + pub sheet_index: u32, + /// Worksheet name as stored by the source format. + pub sheet_name: String, + /// Inclusive source extent that produced the returned table. + pub range: SpreadsheetRange, +} + +impl From for SpreadsheetSource { + fn from(source: model::SpreadsheetSource) -> Self { + SpreadsheetSource { + sheet_index: source.sheet_index, + sheet_name: source.sheet_name, + range: source.range.into(), + } + } +} + /// Canonical table grid: every logical grid position appears exactly once. /// Content and spans live on the origin slot, and each position a span covers /// holds a `covered` slot pointing back at that origin. @@ -362,6 +413,9 @@ pub struct Table { /// Number of leading rows that are header rows (0 = no header). pub header_rows: u32, pub kind: TableKind, + /// Worksheet identity and source extent for a spreadsheet table. + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, } impl From for Table { @@ -377,6 +431,7 @@ impl From for Table { model::TableKind::Data => TableKind::Data, model::TableKind::Layout => TableKind::Layout, }, + source: table.source.map(SpreadsheetSource::from), } } } @@ -428,11 +483,19 @@ pub struct Cell { pub blocks: Vec, pub col_span: u32, pub row_span: u32, + /// Inclusive source range for a spreadsheet origin cell. + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, } impl From for Cell { fn from(cell: model::Cell) -> Self { - Cell { blocks: blocks(cell.blocks), col_span: cell.col_span, row_span: cell.row_span } + Cell { + blocks: blocks(cell.blocks), + col_span: cell.col_span, + row_span: cell.row_span, + source: cell.source.map(SpreadsheetRange::from), + } } } diff --git a/wasm/src/typescript.rs b/wasm/src/typescript.rs index fb6d71e1..f158fbe8 100644 --- a/wasm/src/typescript.rs +++ b/wasm/src/typescript.rs @@ -187,6 +187,29 @@ export type TableKind = /** Layout scaffolding (text boxes, positioning tables). */ | 'layout' +export interface SpreadsheetCoordinate { + /** Zero-based row. */ + row: number + /** Zero-based column. */ + column: number +} + +export interface SpreadsheetRange { + /** Inclusive range start. */ + start: SpreadsheetCoordinate + /** Inclusive range end. */ + end: SpreadsheetCoordinate +} + +export interface SpreadsheetSource { + /** Zero-based position in the source workbook's worksheet order. */ + sheetIndex: number + /** Worksheet name as stored by the source format. */ + sheetName: string + /** Inclusive source extent that produced the returned table. */ + range: SpreadsheetRange +} + /** * Canonical table grid: every logical grid position appears exactly once. * Content and spans live on the origin slot, and each position a span covers @@ -197,6 +220,8 @@ export interface Table { /** Number of leading rows that are header rows (0 = no header). */ headerRows: number kind: TableKind + /** Worksheet identity and source extent for a spreadsheet table. */ + source?: SpreadsheetSource } export type CellSlotKind = 'origin' | 'covered' @@ -215,6 +240,8 @@ export interface Cell { blocks: Array colSpan: number rowSpan: number + /** Inclusive source range for a spreadsheet origin cell. */ + source?: SpreadsheetRange } export type NoteKind = 'footnote' | 'endnote' diff --git a/wasm/test.mjs b/wasm/test.mjs index 86b675ec..5d5d68f7 100644 --- a/wasm/test.mjs +++ b/wasm/test.mjs @@ -20,6 +20,7 @@ initSync({ module: await readFile(fileURLToPath(new URL('./pkg/anydoc_wasm_bg.wa const OUTLINE = await readFile(fixture('docx/handmade-outline.docx')) const RICH = await readFile(fixture('docx/handmade-rich.docx')) +const SPREADSHEET = await readFile(fixture('xlsx/handmade-merged.xlsx')) const CSV = await readFile(fixture('csv/sheet.csv')) const PDF = await readFile(fixture('pdf/text.pdf')) const ENCRYPTED = await readFile(fixture('malformed/encrypted--errors.odt')) @@ -51,6 +52,31 @@ test('toDocument exposes the document model', () => { assert.equal(typeof heading.content[0].style.bold, 'boolean') }) +test('toDocument exposes spreadsheet source coordinates', () => { + const document = toDocument(SPREADSHEET, 'xlsx') + const table = document.blocks.find((block) => block.kind === 'table')?.table + assert.ok(table) + assert.equal(table.source.sheetIndex, 0) + assert.equal(table.source.sheetName, 'Merged') + assert.deepEqual(table.source.range, { + start: { row: 0, column: 0 }, + end: { row: 2, column: 2 }, + }) + assert.deepEqual(table.grid[0][0].cell.source, { + start: { row: 0, column: 0 }, + end: { row: 0, column: 1 }, + }) + assert.deepEqual(table.grid[1][0].cell.source, { + start: { row: 1, column: 0 }, + end: { row: 2, column: 0 }, + }) + + const docx = toDocument(RICH, 'docx') + const docxTable = docx.blocks.find((block) => block.kind === 'table')?.table + assert.ok(docxTable) + assert.equal(docxTable.source, undefined) +}) + test('toDocument carries embedded assets as Uint8Arrays', () => { const document = toDocument(RICH, 'docx') const image = document.assets.find((asset) => asset.mediaType === 'image/png')