From 1c2a3932066ea817868e0c0d37be165c6686b570 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 04:03:17 +0000 Subject: [PATCH] Pin reparse windows at both edges; serialize without a whole-output parse reparseBlocks stitched the old blocks after its window without checking that the edit had not run into them, so an unclosed fence, display-math block or HTML comment was read as one short block followed by the old suffix, where a whole parse has it run to the end of the text. A window now opens at the start of the untouched block before it and must close with the untouched block after it, exactly as that block was; otherwise the reparse reads on to the end. Frontmatter, decided by unbounded lookahead, is refused by a window that opens the note and stops short of its close. serializeDocument uses the same pinned windows to build its next document from the blocks it moved plus small reparses around what changed, instead of parsing its whole output again. On an 8 MB note a one-block save goes from about 130-200 ms to 40-80 ms. Line-ending detection uses two native scans instead of a character loop. Property tests hold both to a whole parse on generated edits and saves (LF, CRLF, BOM, no final newline, no frontmatter). Phase 23, v0.1.20. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011Zsubk1jzejiCfkbLeWVQd --- CHANGELOG.md | 23 ++++ README.md | 7 +- docs/design/contract-v0.md | 13 +- docs/design/roadmap.md | 16 +++ package.json | 2 +- src/backend/native-scanner.ts | 8 +- src/parse.ts | 25 ++-- src/reparse-window.test.ts | 166 ++++++++++++++++++++++++++ src/reparse.ts | 80 ++++++------- src/serialize-incremental.test.ts | 190 ++++++++++++++++++++++++++++++ src/serialize.ts | 154 +++++++++++++++++++++++- src/window.ts | 77 ++++++++++++ 12 files changed, 694 insertions(+), 67 deletions(-) create mode 100644 src/reparse-window.test.ts create mode 100644 src/serialize-incremental.test.ts create mode 100644 src/window.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 461a161..cd78ba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ change the API; each entry says so when it does. Phases refer to [`docs/design/roadmap.md`](docs/design/roadmap.md). +## [0.1.20] - 2026-09-30 + +### Fixed +- `reparseBlocks` and `reparseFromText` no longer cut a block short at the + edge of their window. An edit that opened a code fence, a display-math block + or an HTML comment without closing it was read as one short block followed + by the old blocks after it, where a whole parse has it run to the end of the + text. The window now has to end with the untouched block after it, exactly + as that block was; when it does not, the reparse reads on to the end. A line + that now continues the block before the window (lazy continuation) is also + picked up. (Phase 23.) + +### Changed +- `serializeDocument` builds the next document from the blocks it moved and + small reparses around what changed, pinned the same way, instead of parsing + its whole output again. The document is the one `parseDocument` would + return, which the tests check on thousands of generated saves; the whole + parse remains as the fallback. On an 8 MB note a one-block save goes from + about 130-200 ms to 40-80 ms. (Phase 23.) +- Line-ending detection uses two native scans instead of a character loop, + which took 20-35 ms of every parse of an 8 MB note. + ## [0.1.19] - 2026-09-29 ### Fixed @@ -140,6 +162,7 @@ Phases refer to [`docs/design/roadmap.md`](docs/design/roadmap.md). `@roobli/md/legacy-micromark`. (Phase 6; breaking for callers that relied on the fallback.) +[0.1.20]: https://github.com/roobli/md/compare/v0.1.19...v0.1.20 [0.1.19]: https://github.com/roobli/md/compare/v0.1.18...v0.1.19 [0.1.18]: https://github.com/roobli/md/compare/v0.1.17...v0.1.18 [0.1.17]: https://github.com/roobli/md/compare/v0.1.16...v0.1.17 diff --git a/README.md b/README.md index 5df7557..1bf883d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,10 @@ AGPL. - **Stable spans.** Every block carries its `start` and `end` in the source, its kind, and the literal gap before it. - **Local reparse.** `reparseBlocks` and `reparseFromText` rebuild only the - window around an edit; blocks outside it keep their identity. + window around an edit; blocks outside it keep their identity. The window is + pinned at both edges, so the result is always what a whole parse would say; + an edit that runs past it (an unclosed fence) is read on to the end. + `serializeDocument` builds its next document the same way. - **One dialect.** CommonMark plus GFM tables, task lists and strikethrough, `$`/`$$` math, YAML frontmatter, footnotes, callouts, wiki links and CJK-friendly emphasis. Behaviour is checked against micromark for parity. @@ -45,7 +48,7 @@ Design notes: [vision](docs/design/vision.md) · ## Install ``` -pnpm add github:roobli/md#v0.1.19 +pnpm add github:roobli/md#v0.1.20 ``` Pin a tag: `main` moves. A host that runs install scripts may need to allow diff --git a/docs/design/contract-v0.md b/docs/design/contract-v0.md index b5d3fca..98e3ea2 100644 --- a/docs/design/contract-v0.md +++ b/docs/design/contract-v0.md @@ -84,11 +84,19 @@ only the dirty window (widened by `neighborSlack`), and stitches prefix / middle / suffix with absolute offsets. Untouched prefix spans keep object identity; untouched suffix spans keep `markdown` string identity. Micromark is not imported on this path (Phase 6); use `@roobli/md/legacy-micromark` for compat. +The window is pinned at both edges (Phase 23, `src/window.ts`). It opens at the +start of the untouched block before the dirty ordinals, which is re-read, so a +line that now continues that block is picked up. It must close with the +untouched block after them, exactly as that block was; if the edit ran on into +it (an unclosed fence, display math or comment), the reparse reads on to the +end of the text instead. The result is always what `parseBlocks` of the whole +text would give; `neighborSlack` only trades window size for how often the +fallback is needed, and is no longer what keeps the result correct. + Noto call sites: `replaceMarkdown` can pass the differing middle ordinals as `replacedBlocks` instead of `splitBlocks(fullMarkdown)`; when the host only has the full next buffer, prefer `reparseFromText(prior, nextText)` (Phase 11). -The single-block save path can use `neighborSlack: 0` after `parseSingleBlock` -validation. +`neighborSlack: 0` is safe on every path. ### Serialize (Phase 5) @@ -224,6 +232,7 @@ Full table: [`roadmap.md`](./roadmap.md). Contract-facing summary: | **20** (done) | Empty list item + blank → structural outside | | **21** (done) | List-steal pipe-less delimiters not tables | | **22** (done) | Lazy only with open paragraph / link-def title / ordered start≠1 | +| **23** (done) | Pinned reparse windows; `serializeDocument` without a whole-output parse | ## Replace boundary diff --git a/docs/design/roadmap.md b/docs/design/roadmap.md index f0303ce..79ab6e1 100644 --- a/docs/design/roadmap.md +++ b/docs/design/roadmap.md @@ -313,3 +313,19 @@ Phase 21 **shipped** on `main` (v0.1.18). Phase 22 **shipped** on `main` (v0.1.19). +## Phase 23 acceptance + +- [x] A reparse window is pinned at both edges (`src/window.ts`): it opens at the + start of an untouched block, and must close with the untouched block after + it, exactly as that block was; otherwise it reads on to the end of the text +- [x] Frontmatter, the one block decided by unbounded lookahead, is refused by a + window that opens the note and stops short of its close +- [x] `reparseBlocks` / `reparseFromText` match `parseBlocks` on generated edits, + including unclosed fences, math, comments and lazy continuation + (`reparse-window.test.ts`) +- [x] `serializeDocument` returns the document `parseDocument(outputBytes)` would, + without a whole parse, on generated saves over LF, CRLF, BOM, no final + newline and no frontmatter (`serialize-incremental.test.ts`) +- [x] Package `0.1.20` + +Phase 23 **shipped** on `main` (v0.1.20). diff --git a/package.json b/package.json index a465f04..bd42307 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@roobli/md", - "version": "0.1.19", + "version": "0.1.20", "description": "WYSIWYG-first markdown engine for Noto — block-oriented, byte-exact friendly, fast on large vaults.", "license": "MIT", "type": "module", diff --git a/src/backend/native-scanner.ts b/src/backend/native-scanner.ts index cd18e74..8b7b205 100644 --- a/src/backend/native-scanner.ts +++ b/src/backend/native-scanner.ts @@ -514,8 +514,12 @@ function spansToSplit(text: string, raw: { kind: BlockKind; start: number; end: * Native split for the Phase 1–5 dialect. Always returns a SplitDocument * (never null). Micromark is not consulted — see `@roobli/md/legacy-micromark` * for the Phase 0 compatibility backend. + * + * `atDocumentStart` is false when `text` is a window cut from the middle of a + * document: YAML frontmatter is only recognised at the very start of a file, + * so a `---` that opens a window is a thematic break there, not frontmatter. */ -export function tryNativeSplit(text: string): SplitDocument { +export function tryNativeSplit(text: string, atDocumentStart = true): SplitDocument { if (text.length === 0) { return { spans: [], leading: '', gaps: [], trailing: '' }; } @@ -532,7 +536,7 @@ export function tryNativeSplit(text: string): SplitDocument { } // YAML frontmatter (only at absolute document start) - if (i === 0 && line.start === 0 && isFrontmatterOpen(line.content, true)) { + if (atDocumentStart && i === 0 && line.start === 0 && isFrontmatterOpen(line.content, true)) { let j = i + 1; let closed = false; while (j < lines.length) { diff --git a/src/parse.ts b/src/parse.ts index 06fe815..3bdff57 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -7,16 +7,21 @@ function hasBom(bytes: Uint8Array): boolean { return bytes.length >= 3 && bytes[0] === UTF8_BOM[0] && bytes[1] === UTF8_BOM[1] && bytes[2] === UTF8_BOM[2]; } -function detectLineEnding(text: string): EngineEnvelope['lineEnding'] { - let crlf = 0; - let bareLf = 0; - for (let index = 0; index < text.length; index += 1) { - if (text[index] !== '\n') continue; - if (index > 0 && text[index - 1] === '\r') crlf += 1; - else bareLf += 1; - } - if (crlf > 0 && bareLf > 0) return 'mixed'; - return crlf > 0 ? 'crlf' : 'lf'; +/** An LF with no CR before it. No `g` flag, so the regex carries no state. */ +const BARE_LF = /(?:^|[^\r])\n/; + +/** + * `crlf` when every line ends CRLF, `lf` when none does, `mixed` otherwise. + * + * Two native scans (`includes` and one regex test) rather than a character + * loop: on an 8 MB note this was 20 to 35 ms of every parse and is now well + * under one. + */ +export function detectLineEnding(text: string): EngineEnvelope['lineEnding'] { + const crlf = text.includes('\r\n'); + const bareLf = BARE_LF.test(text); + if (crlf && bareLf) return 'mixed'; + return crlf ? 'crlf' : 'lf'; } function decodeUtf8(bytes: Uint8Array): string | null { diff --git a/src/reparse-window.test.ts b/src/reparse-window.test.ts new file mode 100644 index 0000000..70aaee3 --- /dev/null +++ b/src/reparse-window.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from 'vitest'; +import { parseBlocks } from './parse.js'; +import { reparseFromText } from './reparse.js'; +import type { SplitDocument } from './types.js'; + +/** + * An incremental reparse is only correct if it says what a whole parse of the + * same text says. These hold it to exactly that, including for edits whose + * effect reaches well past the blocks they touched. + */ + +function shape(split: SplitDocument) { + return { + spans: split.spans.map((span) => [span.kind, span.start, span.end, span.markdown]), + leading: split.leading, + gaps: split.gaps, + trailing: split.trailing, + }; +} + +function expectSameAsFull(prior: SplitDocument, next: string, neighborSlack: number, label = ''): void { + const incremental = reparseFromText(prior, next, { neighborSlack }); + expect(shape(incremental), label).toEqual(shape(parseBlocks(next))); +} + +const PARAGRAPHS = Array.from({ length: 12 }, (_, index) => `Paragraph ${index}.`).join('\n\n') + '\n'; + +describe('reparse windows that an edit runs out of', () => { + it('follows an unclosed fence to the end of the note', () => { + const prior = parseBlocks(PARAGRAPHS); + const at = PARAGRAPHS.indexOf('Paragraph 3.'); + const next = `${PARAGRAPHS.slice(0, at)}\`\`\`\n${PARAGRAPHS.slice(at)}`; + for (const slack of [0, 1, 2]) expectSameAsFull(prior, next, slack, `slack ${slack}`); + expect(parseBlocks(next).spans.at(-1)!.kind).toBe('fenced-code'); + }); + + it('follows an unclosed comment, and math, the same way', () => { + const prior = parseBlocks(PARAGRAPHS); + const at = PARAGRAPHS.indexOf('Paragraph 5.'); + for (const opener of ['', '
\n', '
\n', '- ', '* ', '1. ', '> ', '# ', + '---\n', '===\n', ' ', '\n', '\n\n', '| x | y |\n| - | - |\n', '[r]: /u\n', '[^n]: note\n', 'word ', 'text\n', +]; + +describe('reparse windows, against a whole parse, on generated edits', () => { + for (const slack of [0, 1]) { + it(`agrees with parseBlocks on 600 random edits (slack ${slack})`, () => { + const prior = parseBlocks(SOURCE); + for (let seed = 1; seed <= 600; seed += 1) { + const random = rng(seed * 7 + slack); + const at = Math.floor(random() * (SOURCE.length + 1)); + const pick = random(); + let next: string; + if (pick < 0.55) { + const snippet = SNIPPETS[Math.floor(random() * SNIPPETS.length)]!; + next = SOURCE.slice(0, at) + snippet + SOURCE.slice(at); + } else if (pick < 0.85) { + const end = Math.min(SOURCE.length, at + 1 + Math.floor(random() * 40)); + next = SOURCE.slice(0, at) + SOURCE.slice(end); + } else { + const end = Math.min(SOURCE.length, at + 1 + Math.floor(random() * 20)); + const snippet = SNIPPETS[Math.floor(random() * SNIPPETS.length)]!; + next = SOURCE.slice(0, at) + snippet + SOURCE.slice(end); + } + expectSameAsFull(prior, next, slack, `seed ${seed}`); + } + }); + } +}); diff --git a/src/reparse.ts b/src/reparse.ts index 40663fd..50b5126 100644 --- a/src/reparse.ts +++ b/src/reparse.ts @@ -12,6 +12,7 @@ */ import { joinSplit, parseBlocks } from './parse.js'; +import { parseWindow } from './window.js'; import type { BlockSpan, SplitDocument } from './types.js'; /** Character edit against the prior joined source (`joinSplit(prior)`). */ @@ -202,18 +203,6 @@ function resolveDirtyOrdinals( return { from, to }; } -function priorWindow( - prior: SplitDocument, - priorText: string, - dirtyFrom: number, - dirtyTo: number, -): { start: number; end: number } { - const n = prior.spans.length; - const start = dirtyFrom === 0 ? 0 : prior.spans[dirtyFrom - 1]!.end; - const end = dirtyTo === n - 1 ? priorText.length : prior.spans[dirtyTo + 1]!.start; - return { start, end }; -} - function shiftSpan(span: BlockSpan, delta: number): BlockSpan { if (delta === 0) return span; return { @@ -225,16 +214,6 @@ function shiftSpan(span: BlockSpan, delta: number): BlockSpan { }; } -function remapLocalSpan(span: BlockSpan, windowStart: number): BlockSpan { - return { - kind: span.kind, - start: span.start + windowStart, - end: span.end + windowStart, - markdown: span.markdown, - node: span.node, - }; -} - function stitchCoverage(text: string, spans: readonly BlockSpan[], prior: SplitDocument, dirtyFrom: number, dirtyTo: number, middleCount: number): SplitDocument { if (spans.length === 0) { return { spans: [], leading: '', gaps: [], trailing: text }; @@ -311,34 +290,47 @@ export function reparseBlocks(options: ReparseBlocksOptions): ReparseBlocksResul const dirty = resolveDirtyOrdinals(prior, edit, replacedBlocks, slack); if (dirty === 'full' || prior.spans.length === 0) return finishFull(); - const { from: dirtyFrom, to: dirtyTo } = dirty; - const window = priorWindow(prior, priorText, dirtyFrom, dirtyTo); - - let windowStart: number; - let windowEnd: number; - if (edit) { - windowStart = mapPriorOffsetExclusiveEnd(window.start, edit); - windowEnd = mapPriorOffsetExclusiveEnd(window.end, edit); - } else { - // Block-ordinal path: host guarantees the change is confined to the dirty - // window, so prefix bytes are unchanged and suffix length is preserved. - windowStart = window.start; - windowEnd = text.length - (priorText.length - window.end); - } + const n = prior.spans.length; + // Everything before the edit is where it was; everything after it moved by + // the same amount. On the block path the host vouches that the change sits + // inside the dirty ordinals. + const delta = edit + ? edit.inserted.length - (edit.priorEnd - edit.priorStart) + : text.length - priorText.length; - if (windowStart < 0 || windowEnd < windowStart || windowEnd > text.length) { + // The window is pinned at both edges (see `window.ts`): it opens at the start + // of the untouched block before the dirty ordinals, and must close with the + // untouched block after them, exactly as that block was. If the edit ran on + // into it, as an unclosed fence does, the window is read on to the end of + // the text instead. + const leftOrdinal = dirty.from - 1; + const rightOrdinal = dirty.to + 1; + const left = leftOrdinal >= 0 ? prior.spans[leftOrdinal]! : null; + const right = rightOrdinal < n ? shiftSpan(prior.spans[rightOrdinal]!, delta) : null; + const windowStart = left ? left.start : 0; + if ( + (left && text.slice(left.start, left.end) !== left.markdown) + || (right && (right.end > text.length || text.slice(right.start, right.end) !== right.markdown)) + ) { return finishFull(); } - const slice = text.slice(windowStart, windowEnd); - const local = parseBlocks(slice); - const middle = local.spans.map((span) => remapLocalSpan(span, windowStart)); + let local = right ? parseWindow(text, windowStart, right.end, right) : null; + const anchored = local !== null; + if (local === null) local = parseWindow(text, windowStart, text.length, null); + if (local === null) return finishFull(); + const windowEnd = anchored ? right!.end : text.length; + + // The block before the window keeps its identity unless the edit changed how + // it ends (a line that now continues it, say). + const leftUnchanged = left !== null && local.length > 0 + && local[0]!.start === left.start && local[0]!.end === left.end && local[0]!.kind === left.kind; + const dirtyFrom = leftUnchanged ? leftOrdinal + 1 : Math.max(0, leftOrdinal); + const dirtyTo = anchored ? rightOrdinal - 1 : n - 1; + const middle = local.slice(leftUnchanged ? 1 : 0, anchored ? -1 : undefined); const prefix = prior.spans.slice(0, dirtyFrom) as BlockSpan[]; - const delta = edit - ? edit.inserted.length - (edit.priorEnd - edit.priorStart) - : text.length - priorText.length; - const suffix = prior.spans.slice(dirtyTo + 1).map((span) => shiftSpan(span, delta)); + const suffix = anchored ? prior.spans.slice(rightOrdinal).map((span) => shiftSpan(span, delta)) : []; const spans: BlockSpan[] = [...prefix, ...middle, ...suffix]; const stitched = stitchCoverage(text, spans, prior, dirtyFrom, dirtyTo, middle.length); diff --git a/src/serialize-incremental.test.ts b/src/serialize-incremental.test.ts new file mode 100644 index 0000000..4f46765 --- /dev/null +++ b/src/serialize-incremental.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from 'vitest'; +import { parseDocument } from './parse.js'; +import { identityUnits, serializeDocument, type SerializeUnit } from './serialize.js'; +import type { EngineDocument } from './types.js'; + +/** + * A save builds the next document from the blocks it moved and a few small + * reparses instead of parsing its whole output again. That is only sound if + * the document it builds is the one a parse of the output would give; these + * hold it to exactly that. + */ + +const encoder = new TextEncoder(); + +function parsed(text: string, bom = false): EngineDocument { + const body = encoder.encode(text); + const bytes = bom ? Uint8Array.from([0xef, 0xbb, 0xbf, ...body]) : body; + const result = parseDocument(bytes); + if (result.status !== 'parsed') throw new Error(result.message); + return result.document; +} + +function shape(document: EngineDocument) { + return { + envelope: document.envelope, + text: document.text, + leading: document.leading, + trailing: document.trailing, + gaps: document.gaps, + blocks: document.blocks.map((block) => [block.kind, block.start, block.end, block.markdown, block.node]), + }; +} + +/** Serialize, then check the document against a parse of the output. */ +function expectFaithful(document: EngineDocument, units: SerializeUnit[], label: string, envelope?: { hasFinalNewline?: boolean }) { + const result = envelope ? serializeDocument(document, { units, envelope }) : serializeDocument(document, { units }); + if (result.status !== 'serialized') return result; + const full = parseDocument(result.outputBytes); + if (full.status !== 'parsed') throw new Error(full.message); + expect(shape(result.document), label).toEqual(shape(full.document)); + return result; +} + +const SOURCE = `--- +title: Saves +--- + +# Title + +A paragraph with *emphasis* and a [link](https://example.com). +It continues on a second line. + +- first item +- second item + - nested item + +Between the lists. + +- another list + +1. one +2. two + +> A quote +> that goes on. + +\`\`\`ts +const value = 1; + +const other = 2; +\`\`\` + +| a | b | +| - | - | +| 1 | 2 | + +$$ +x = y^2 +$$ + +
html
+ +Text with a footnote.[^f] + +[^f]: The footnote. + +[ref]: https://example.com/ref + +Setext heading +-------------- + + indented code + +*** + +Last paragraph. +`; + +const REPLACEMENTS = [ + 'A plain paragraph.', '```\nnever closed', '$$\nnever closed', '