diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs index 686b59b3c..a80d11ce5 100644 --- a/crates/utopia-ingest/src/parsers.rs +++ b/crates/utopia-ingest/src/parsers.rs @@ -7,6 +7,11 @@ use std::io::{Cursor, Read, Write}; use std::process::Command; const SPREADSHEET_ROW_LIMIT: usize = 2_000; +// Each retained cell passes through a 40-byte GridRow entry and two 48-byte renderer +// Cells, besides source values, strings and Vec spare capacity (~370 B/cell in a +// full dense import). Bound actual records/segments, not their empty column spans: +// a 2,000 x 150 sheet uses 300,000 entries and fits without losing any rows. +const XLSX_SEGMENT_LIMIT: usize = 1_000_000; const CSV_RECORD_LIMIT: usize = 10_000; /// 文本解码:chardetng 探测编码(覆盖 GBK/GB18030/BIG5 等中文常见编码)。 @@ -773,53 +778,200 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<(String, Vec { + let Ok(sheet) = xlsx_sheet(book, &sheet_name, XLSX_SEGMENT_LIMIT) else { + continue; + }; + sheet + } + _ => { + let Ok(range) = workbook.worksheet_range(&sheet_name) else { + continue; + }; + let merges = match &mut workbook { + calamine::Sheets::Xls(book) => book + .merge_cells_by_sheet_name(&sheet_name) + .unwrap_or_default(), + _ => Vec::new(), + }; + let (rows, headers) = spreadsheet_grid(&range, &merges); + let text = crate::table::render_grid_with_headers(&rows, headers) + .map(|text| format!("{text}\n")) + .unwrap_or_else(|| { + let mut text = String::new(); + for row in &rows { + let cells: Vec<&str> = row.iter().map(|(s, _, _)| s.as_str()).collect(); + text.push_str(&cells.join("\t")); + text.push('\n'); + } + text + }); + ( + text, + range.height().min(SPREADSHEET_ROW_LIMIT), + range.height(), + ) + } }; - if range.is_empty() { + if rows_read == 0 { continue; } - // xlsx/xls 把 merge 单独给出;xlsb/ods 的 calamine reader 没有这一步,按无 merge 读。 - let merges = match &mut workbook { - calamine::Sheets::Xlsx(book) => book - .merge_cells_by_sheet_name(&sheet_name) - .unwrap_or_default(), - calamine::Sheets::Xls(book) => book - .merge_cells_by_sheet_name(&sheet_name) - .unwrap_or_default(), - _ => Vec::new(), - }; out.push_str(&format!("\n# Sheet: {sheet_name}\n\n")); - let rows_total = range.height(); - if rows_total > SPREADSHEET_ROW_LIMIT { - let rows_read = SPREADSHEET_ROW_LIMIT; + if rows_total > rows_read { warnings.push(crate::ParseWarning { kind: crate::ParseWarning::SPREADSHEET_ROWS_TRUNCATED, detail: serde_json::json!({ - "sheet": sheet_name.clone(), + "sheet": sheet_name, "rows_read": rows_read, "rows_total": rows_total, "rows_omitted": rows_total - rows_read, }), }); } - // 每张表按网格渲染成带列头的 Markdown 表;一格字都没有的表退回制表符分隔 - let (rows, headers) = spreadsheet_grid(&range, &merges); - match crate::table::render_grid_with_headers(&rows, headers) { - Some(md) => { - out.push_str(&md); - out.push('\n'); + out.push_str(&text); + } + Ok((out, warnings)) +} + +fn xlsx_sheet( + book: &mut calamine::Xlsx>>, + sheet: &str, + budget: usize, +) -> anyhow::Result<(String, usize, usize)> { + let mut start_row = u32::MAX; + let mut end_row = 0; + let mut row_counts = vec![0usize; SPREADSHEET_ROW_LIMIT]; + let mut reader = book.worksheet_cells_reader(sheet)?; + // Keep only the first 2,000 physical rows' counts, even if XML cells arrive + // out of order. The full row bounds survive so truncation remains honest. + while let Some(cell) = reader.next_cell()? { + if cell.get_value() == &calamine::DataRef::Empty { + continue; + } + let (row, col) = cell.get_position(); + anyhow::ensure!( + row < 1_048_576 && col < 16_384, + "Cell exceeds the Excel worksheet limits" + ); + if row < start_row { + let shift = (start_row - row).min(SPREADSHEET_ROW_LIMIT as u32) as usize; + row_counts.rotate_right(shift); + row_counts[..shift].fill(0); + start_row = row; + } + end_row = end_row.max(row); + let offset = (row - start_row) as usize; + if let Some(count) = row_counts.get_mut(offset) { + // Repeated coordinates overwrite values below, so even malformed + // XML cannot retain more than one physical row's worth of cells. + *count = (*count + 1).min(16_384); + } + } + drop(reader); + if start_row == u32::MAX { + return Ok((String::new(), 0, 0)); + } + let rows_total = (end_row - start_row + 1) as usize; + let mut height = rows_total.min(SPREADSHEET_ROW_LIMIT); + let mut count = 0; + for (row, &cells) in row_counts[..height].iter().enumerate() { + if cells > budget - count { + height = row; + break; + } + count += cells; + } + // Counts bound this allocation before any source values are retained. A + // distant note adds one entry, not thousands of empty Data/String objects. + let mut values = vec![std::collections::BTreeMap::::new(); height]; + let mut reader = book.worksheet_cells_reader(sheet)?; + while let Some(cell) = reader.next_cell()? { + let (row, col) = cell.get_position(); + let offset = row.saturating_sub(start_row) as usize; + if row >= start_row && offset < height && cell.get_value() != &calamine::DataRef::Empty { + let text = spreadsheet_cell_text(Some(&calamine::Data::from(cell.get_value().clone()))); + values[offset].insert(col, text); + } + } + drop(reader); + let merges = book.merge_cells_by_sheet_name(sheet).unwrap_or_default(); + loop { + let start_col = values + .iter() + .filter_map(|row| row.keys().next()) + .min() + .copied() + .unwrap(); + let end_col = values + .iter() + .filter_map(|row| row.keys().next_back()) + .max() + .copied() + .unwrap(); + let width = (end_col - start_col + 1) as usize; + let text_at = |row: usize, col: usize| { + values[row] + .get(&(start_col + col as u32)) + .cloned() + .unwrap_or_default() + }; + let regions = spreadsheet_regions( + (start_row, start_col), + (start_row + height as u32 - 1, end_col), + &merges, + text_at, + ); + let mut owner = vec![usize::MAX; width]; + let mut rows = Vec::with_capacity(height); + let mut segments = 0; + for row in 0..height { + let cells = spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + true, + ); + if cells.len() > budget - segments { + break; } + segments += cells.len(); + rows.push(cells); + } + if rows.len() != height { + height = rows.len(); + values.truncate(height); + // Dropped rows cannot leave a distant column in the kept window's + // geometry. Reproject merges too, retaining their physical spans. + continue; + } + let headers = spreadsheet_headers(®ions, &rows); + let text = match crate::table::render_grid_with_headers(&rows, headers) { + Some(text) => format!("{text}\n"), None => { - for line in &rows { - let cells: Vec<&str> = line.iter().map(|(s, _, _)| s.as_str()).collect(); - out.push_str(&cells.join("\t")); - out.push('\n'); + // Preserve the old TSV fallback exactly, but materialize only + // one dense row at a time rather than the whole rectangle. + let mut text = String::new(); + for row in 0..height { + let cells = spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + false, + ); + let cells: Vec<&str> = cells.iter().map(|(s, _, _)| s.as_str()).collect(); + text.push_str(&cells.join("\t")); + text.push('\n'); } + text } - } + }; + return Ok((text, height, rows_total)); } - Ok((out, warnings)) } #[derive(Clone)] @@ -843,9 +995,36 @@ fn spreadsheet_grid( if height == 0 || width == 0 { return (Vec::new(), None); } - let end_row = start_row + height as u32 - 1; - let end_col = start_col + width as u32 - 1; + let text_at = |row, col| spreadsheet_cell_text(range.get((row, col))); + let regions = spreadsheet_regions( + (start_row, start_col), + (start_row + height as u32 - 1, start_col + width as u32 - 1), + merges, + text_at, + ); + let mut owner = vec![usize::MAX; width]; + let rows: Vec<_> = (0..height) + .map(|row| { + spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + false, + ) + }) + .collect(); + let headers = spreadsheet_headers(®ions, &rows); + (rows, headers) +} +fn spreadsheet_regions( + (start_row, start_col): (u32, u32), + (end_row, end_col): (u32, u32), + merges: &[calamine::Dimensions], + text_at: impl Fn(usize, usize) -> String, +) -> Vec { let mut regions = Vec::new(); for merge in merges { if merge.start.0 > merge.end.0 @@ -868,30 +1047,40 @@ fn spreadsheet_grid( regions.push(GridMerge { start, end, - text: spreadsheet_cell_text(range.get(start)), + text: text_at(start.0, start.1), }); } + regions +} - let mut owner = vec![usize::MAX; height * width]; +fn spreadsheet_row( + row: usize, + width: usize, + regions: &[GridMerge], + owner: &mut [usize], + text_at: impl Fn(usize) -> String, + compact_empty: bool, +) -> GridRow { + // One owner row keeps the old last-merge-wins projection and anchor jumps, + // without allocating height * width indices. XLSX bounds width at XFD. + owner.fill(usize::MAX); for (index, region) in regions.iter().enumerate() { - for row in region.start.0..=region.end.0 { - for col in region.start.1..=region.end.1 { - owner[row * width + col] = index; - } + if region.start.0 <= row && row <= region.end.0 { + owner[region.start.1..=region.end.1].fill(index); } } - - let mut rows: Vec = Vec::with_capacity(height); - for row in 0..height { - let mut cells = GridRow::new(); - let mut col = 0usize; - while col < width { - let index = owner[row * width + col]; - if index == usize::MAX { - cells.push((spreadsheet_cell_text(range.get((row, col))), 1, 0)); - col += 1; - continue; + let mut cells: GridRow = Vec::new(); + let mut col = 0; + while col < width { + let index = owner[col]; + if index == usize::MAX { + let text = text_at(col); + match (compact_empty && text.is_empty(), cells.last_mut()) { + (true, Some((previous, span, _))) if previous.is_empty() => *span += 1, + _ => cells.push((text, 1, 0)), } + col += 1; + } else { let region = ®ions[index]; if col == region.start.1 { cells.push((region.text.clone(), region.end.1 - region.start.1 + 1, 0)); @@ -900,11 +1089,8 @@ fn spreadsheet_grid( col += 1; } } - rows.push(cells); } - - let headers = spreadsheet_headers(®ions, &rows); - (rows, headers) + cells } fn spreadsheet_cell_text(cell: Option<&calamine::Data>) -> String { @@ -1205,6 +1391,95 @@ fn pptx_xml_to_text(xml: &str) -> anyhow::Result { mod tests { use super::*; + fn budget_book(sheet_data: &str, merges: &str) -> calamine::Xlsx>> { + let worksheet = format!( + r#"{sheet_data}{merges}"# + ); + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (name, xml) in [ + ( + "[Content_Types].xml", + r#""#, + ), + ( + "_rels/.rels", + r#""#, + ), + ( + "xl/workbook.xml", + r#""#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#""#, + ), + ("xl/worksheets/sheet1.xml", &worksheet), + ] { + zip.start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(xml.as_bytes()).unwrap(); + } + calamine::open_workbook_from_rs(Cursor::new(zip.finish().unwrap().into_inner())).unwrap() + } + + #[test] + fn a_segment_budget_keeps_only_complete_rows() { + // Four source cells fit; two rows of three emitted segments do not. + let rows = r#"leftright12"#; + let (_, rows_read, rows_total) = + xlsx_sheet(&mut budget_book(rows, ""), "Budget", 5).unwrap(); + assert_eq!((rows_read, rows_total), (1, 2)); + } + + #[test] + fn dropping_a_row_reprojects_the_retained_title_and_merged_span() { + let title = r#"Title"#; + let rest = r#" note"#; + let merges = r#""#; + // Four source cells fit, but the first two rows use 3 + 4 segments. + let (text, rows_read, rows_total) = xlsx_sheet( + &mut budget_book(&format!("{title}{rest}"), merges), + "Budget", + 5, + ) + .unwrap(); + let (expected, _, _) = xlsx_sheet(&mut budget_book(title, merges), "Budget", 5).unwrap(); + assert_eq!((rows_read, rows_total), (1, 3)); + assert!(text.contains("Title")); + assert_eq!(text, expected); + } + + #[test] + fn empty_runs_keep_the_physical_cell_and_merge_coordinates() { + let regions = [GridMerge { + start: (0, 2), + end: (0, 4), + text: "Merged".into(), + }]; + let row = spreadsheet_row( + 0, + 10, + ®ions, + &mut [usize::MAX; 10], + |col| match col { + 0 => "Item".into(), + 7 => "3".into(), + _ => String::new(), + }, + true, + ); + let expected = [ + ("Item", 1), + ("", 1), + ("Merged", 3), + ("", 2), + ("3", 1), + ("", 2), + ] + .map(|(text, span)| (text.to_owned(), span, 0)); + assert_eq!(row, expected); + } + const DOC: &str = r#" Segment results SegmentRevenue diff --git a/crates/utopia-ingest/tests/spreadsheet_truncation.rs b/crates/utopia-ingest/tests/spreadsheet_truncation.rs index e4f8b7cde..d4df2fd24 100644 --- a/crates/utopia-ingest/tests/spreadsheet_truncation.rs +++ b/crates/utopia-ingest/tests/spreadsheet_truncation.rs @@ -4,7 +4,7 @@ use std::fmt::Write as _; use std::io::{Cursor, Write}; -fn workbook(data_rows: usize) -> Vec { +fn ledger_rows(data_rows: usize) -> String { let mut sheet_data = String::from( r#"idname"#, ); @@ -14,17 +14,69 @@ fn workbook(data_rows: usize) -> Vec { r#"{id}customer-{id}"# )); } - let worksheet = format!( - r#"{sheet_data}"# + sheet_data +} + +fn workbook(data_rows: usize) -> Vec { + workbook_xml(&ledger_rows(data_rows)) +} + +fn workbook_xml(sheet_data: &str) -> Vec { + workbook_sheets(&[("Ledger", sheet_data)]) +} + +fn workbook_sheets(sheets: &[(&str, &str)]) -> Vec { + workbook_sheets_with_chart(sheets, false, "") +} + +fn workbook_sheets_with_chart( + sheets: &[(&str, &str)], + chart: bool, + worksheet_suffix: &str, +) -> Vec { + let mut content_types = String::from( + r#""#, ); + let mut sheet_entries = String::new(); + let mut sheet_rels = String::new(); + let mut parts = Vec::new(); + for (i, (name, sheet_data)) in sheets.iter().enumerate() { + let id = i + 1; + write!(content_types, r#""#).unwrap(); + write!( + sheet_entries, + r#""# + ) + .unwrap(); + write!(sheet_rels, r#""#).unwrap(); + parts.push((format!("xl/worksheets/sheet{id}.xml"), format!(r#"{sheet_data}{worksheet_suffix}"#))); + } + if chart { + let id = sheets.len() + 1; + write!(content_types, r#""#).unwrap(); + write!( + sheet_entries, + r#""# + ) + .unwrap(); + write!(sheet_rels, r#""#).unwrap(); + parts.extend([ + (format!("xl/chartsheets/sheet{id}.xml"), r#""#.into()), + (format!("xl/chartsheets/_rels/sheet{id}.xml.rels"), r#""#.into()), + ("xl/drawings/drawing1.xml".into(), r#""#.into()), + ("xl/drawings/_rels/drawing1.xml.rels".into(), r#""#.into()), + ("xl/charts/chart1.xml".into(), r#"Series 1ItemGeneral1"#.into()), + ]); + } + content_types.push_str(""); + parts.extend([ + ("[Content_Types].xml".into(), content_types), + ("_rels/.rels".into(), r#""#.into()), + ("xl/workbook.xml".into(), format!(r#"{sheet_entries}"#)), + ("xl/_rels/workbook.xml.rels".into(), format!(r#"{sheet_rels}"#)), + ]); let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); - for (part, xml) in [ - ("[Content_Types].xml", r#""#.to_string()), - ("_rels/.rels", r#""#.to_string()), - ("xl/workbook.xml", r#""#.to_string()), - ("xl/_rels/workbook.xml.rels", r#""#.to_string()), - ("xl/worksheets/sheet1.xml", worksheet), - ] { + for (part, xml) in parts { zip.start_file(part, zip::write::SimpleFileOptions::default()) .unwrap(); zip.write_all(xml.as_bytes()).unwrap(); @@ -32,6 +84,23 @@ fn workbook(data_rows: usize) -> Vec { zip.finish().unwrap().into_inner() } +fn words(cell: &str, text: &str) -> String { + format!(r#"{text}"#) +} + +fn column(mut col: usize) -> String { + let mut letters = Vec::new(); + loop { + letters.push(b'A' + (col % 26) as u8); + if col < 26 { + break; + } + col = col / 26 - 1; + } + letters.reverse(); + String::from_utf8(letters).unwrap() +} + fn csv(records: usize) -> String { let mut csv = String::from("id,name\n"); for id in 1..=records { @@ -78,6 +147,126 @@ fn a_spreadsheet_at_the_row_cap_has_no_warning() { assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); } +#[test] +fn a_distant_cell_past_the_row_window_does_not_shrink_the_retained_ledger() { + let mut rows = ledger_rows(1_999); + rows.push_str(&format!( + r#"{}"#, + words("P1000000", "omitted-distant-note") + )); + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(parsed.text.contains("| 1999 | customer-1999 |")); + assert!(!parsed.text.contains("omitted-distant-note")); + assert_eq!(parsed.warnings.len(), 1); + assert_eq!(parsed.warnings[0].kind, "spreadsheet.rows_truncated"); + assert_eq!( + parsed.warnings[0].detail, + serde_json::json!({ + "sheet": "Ledger", "rows_read": 2000, + "rows_total": 1000000, "rows_omitted": 998000, + }) + ); +} + +#[test] +fn a_header_note_beyond_empty_columns_does_not_truncate_a_ledger() { + let rows = ledger_rows(1_500); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(expected.text.contains("| 1500 | customer-1500 |")); + for reference in ["AZZ1", "XFD1"] { + let with_note = rows.replacen( + "", + &format!("{}", words(reference, "header-only-note")), + 1, + ); + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&with_note)).unwrap(); + assert!( + parsed.text == expected.text, + "note at {reference} changed the ledger" + ); + assert!( + parsed.warnings.is_empty(), + "{reference}: {:?}", + parsed.warnings + ); + } +} + +#[test] +fn an_ordinary_dense_sheet_retains_two_thousand_rows_and_one_hundred_fifty_columns() { + let columns: Vec<_> = (0..150).map(column).collect(); + let mut rows = String::from(r#""#); + for (col, name) in columns.iter().enumerate() { + rows.push_str(&words(&format!("{name}1"), &format!("column-{col}"))); + } + rows.push_str(""); + for row in 2..=2_000 { + write!( + rows, + r#"{}"#, + words(&format!("A{row}"), &format!("customer-{}", row - 1)) + ) + .unwrap(); + for (col, name) in columns.iter().enumerate().skip(1) { + write!( + rows, + r#"{}"#, + row * 1_000 + col + ) + .unwrap(); + } + rows.push_str(""); + } + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); + let last = parsed + .text + .lines() + .find(|line| line.starts_with("| customer-1999 |")) + .expect("the final row is retained"); + assert!(last.ends_with("| 2000149 |"), "the last column is retained"); + assert_eq!(last.split('|').count(), 152, "all 150 columns are present"); +} + +#[test] +fn a_broken_sheet_does_not_prevent_reading_the_other_sheets() { + let good = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&good)).unwrap(); + let broken = r#"not-a-number"#; + let bytes = workbook_sheets(&[("Ledger", &good), ("Broken", broken)]); + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + +#[test] +fn unreadable_merges_do_not_prevent_reading_the_worksheet_cells() { + use calamine::Reader as _; + + let rows = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + let merges = r#""#; + let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], false, merges); + let mut book: calamine::Xlsx<_> = + calamine::open_workbook_from_rs(Cursor::new(bytes.clone())).unwrap(); + assert_eq!(book.worksheet_range("Ledger").unwrap().height(), 2); + assert!(book.merge_cells_by_sheet_name("Ledger").is_err()); + + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + +#[test] +fn a_chartsheet_does_not_prevent_reading_the_worksheets() { + let rows = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], true, ""); + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + #[test] fn a_csv_past_the_record_cap_reports_the_records_it_left_out() { let parsed = utopia_ingest::parse("customers.csv", csv(10_000).as_bytes()).unwrap();