From 078b8d5c0a39a6862e9d079d4a744948bc09a069 Mon Sep 17 00:00:00 2001 From: dada-yan Date: Fri, 16 Oct 2026 22:04:37 +1100 Subject: [PATCH] A split paragraph's first chunk carries its headings once Signed-off-by: dada-yan --- crates/utopia-ingest/src/chunker.rs | 32 ++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/utopia-ingest/src/chunker.rs b/crates/utopia-ingest/src/chunker.rs index bbbb625c5..74ba59d2d 100644 --- a/crates/utopia-ingest/src/chunker.rs +++ b/crates/utopia-ingest/src/chunker.rs @@ -422,6 +422,8 @@ impl<'a> Packer<'a> { for (offset, piece) in splitter.chunk_indices(block) { let sub = range.start + offset..range.start + offset + piece.len(); if first { + // 第一小段带着标题当正文,前缀用上面那份:标题还在 `pending` 里时算的,不含它们。 + // 从前每一小段都重算前缀,那时 `pending` 已经倒空,第一块把标题念两遍,有时还超了预算 self.body = headings .iter() .map(|h| match h { @@ -430,9 +432,11 @@ impl<'a> Packer<'a> { }) .collect(); first = false; + } else { + // 续块:标题已经贴上正文,前缀给整条路径 + self.prefix = self.prefix_now(); } self.body.push(Piece::Text(sub)); - self.prefix = self.prefix_now(); self.flush(); } } @@ -800,6 +804,32 @@ The tenant pays rent monthly. } } + /// 小节下的第一段一段就超预算、退回按句切:切开的第一块是「上一级(前缀)· 这一节的标题 · + /// 第一小段」,每条标题一遍;续块照旧以整条路径开头 + #[test] + fn the_first_chunk_of_a_split_paragraph_carries_its_headings_once() { + let text = format!( + "# Report\n\nIntro.\n\n## Part A\n\n{}", + "One sentence here. Another sentence follows it. ".repeat(30) + ); + let pieces = chunk_with_budget(&text, 40); + let i = pieces + .iter() + .position(|p| p.text.contains("One sentence")) + .expect("长段落切出的第一块"); + let source = |p: &ChunkPiece| &text[p.char_start as usize..p.char_end as usize]; + assert_eq!( + pieces[i].text, + format!("# Report\n\n{}", source(&pieces[i])), + "{} token,预算 40", + tokens(&pieces[i].text) + ); + assert_eq!( + pieces[i + 1].text, + format!("# Report\n\n## Part A\n\n{}", source(&pieces[i + 1])) + ); + } + #[test] fn chunks_are_numbered_from_zero_and_never_empty() { let text = "# T\n\nA.\n\n* * *\n\nB.\n\n| x | y |\n| --- | --- |\n| 1 | 2 |";