diff --git a/CHANGELOG.md b/CHANGELOG.md index 461a161..cd78ba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ change the API; each entry says so when it does. Phases refer to [`docs/design/roadmap.md`](docs/design/roadmap.md). +## [0.1.20] - 2026-09-30 + +### Fixed +- `reparseBlocks` and `reparseFromText` no longer cut a block short at the + edge of their window. An edit that opened a code fence, a display-math block + or an HTML comment without closing it was read as one short block followed + by the old blocks after it, where a whole parse has it run to the end of the + text. The window now has to end with the untouched block after it, exactly + as that block was; when it does not, the reparse reads on to the end. A line + that now continues the block before the window (lazy continuation) is also + picked up. (Phase 23.) + +### Changed +- `serializeDocument` builds the next document from the blocks it moved and + small reparses around what changed, pinned the same way, instead of parsing + its whole output again. The document is the one `parseDocument` would + return, which the tests check on thousands of generated saves; the whole + parse remains as the fallback. On an 8 MB note a one-block save goes from + about 130-200 ms to 40-80 ms. (Phase 23.) +- Line-ending detection uses two native scans instead of a character loop, + which took 20-35 ms of every parse of an 8 MB note. + ## [0.1.19] - 2026-09-29 ### Fixed @@ -140,6 +162,7 @@ Phases refer to [`docs/design/roadmap.md`](docs/design/roadmap.md). `@roobli/md/legacy-micromark`. (Phase 6; breaking for callers that relied on the fallback.) +[0.1.20]: https://github.com/roobli/md/compare/v0.1.19...v0.1.20 [0.1.19]: https://github.com/roobli/md/compare/v0.1.18...v0.1.19 [0.1.18]: https://github.com/roobli/md/compare/v0.1.17...v0.1.18 [0.1.17]: https://github.com/roobli/md/compare/v0.1.16...v0.1.17 diff --git a/README.md b/README.md index 5df7557..1bf883d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,10 @@ AGPL. - **Stable spans.** Every block carries its `start` and `end` in the source, its kind, and the literal gap before it. - **Local reparse.** `reparseBlocks` and `reparseFromText` rebuild only the - window around an edit; blocks outside it keep their identity. + window around an edit; blocks outside it keep their identity. The window is + pinned at both edges, so the result is always what a whole parse would say; + an edit that runs past it (an unclosed fence) is read on to the end. + `serializeDocument` builds its next document the same way. - **One dialect.** CommonMark plus GFM tables, task lists and strikethrough, `$`/`$$` math, YAML frontmatter, footnotes, callouts, wiki links and CJK-friendly emphasis. Behaviour is checked against micromark for parity. @@ -45,7 +48,7 @@ Design notes: [vision](docs/design/vision.md) · ## Install ``` -pnpm add github:roobli/md#v0.1.19 +pnpm add github:roobli/md#v0.1.20 ``` Pin a tag: `main` moves. A host that runs install scripts may need to allow diff --git a/docs/design/contract-v0.md b/docs/design/contract-v0.md index b5d3fca..98e3ea2 100644 --- a/docs/design/contract-v0.md +++ b/docs/design/contract-v0.md @@ -84,11 +84,19 @@ only the dirty window (widened by `neighborSlack`), and stitches prefix / middle / suffix with absolute offsets. Untouched prefix spans keep object identity; untouched suffix spans keep `markdown` string identity. Micromark is not imported on this path (Phase 6); use `@roobli/md/legacy-micromark` for compat. +The window is pinned at both edges (Phase 23, `src/window.ts`). It opens at the +start of the untouched block before the dirty ordinals, which is re-read, so a +line that now continues that block is picked up. It must close with the +untouched block after them, exactly as that block was; if the edit ran on into +it (an unclosed fence, display math or comment), the reparse reads on to the +end of the text instead. The result is always what `parseBlocks` of the whole +text would give; `neighborSlack` only trades window size for how often the +fallback is needed, and is no longer what keeps the result correct. + Noto call sites: `replaceMarkdown` can pass the differing middle ordinals as `replacedBlocks` instead of `splitBlocks(fullMarkdown)`; when the host only has the full next buffer, prefer `reparseFromText(prior, nextText)` (Phase 11). -The single-block save path can use `neighborSlack: 0` after `parseSingleBlock` -validation. +`neighborSlack: 0` is safe on every path. ### Serialize (Phase 5) @@ -224,6 +232,7 @@ Full table: [`roadmap.md`](./roadmap.md). Contract-facing summary: | **20** (done) | Empty list item + blank → structural outside | | **21** (done) | List-steal pipe-less delimiters not tables | | **22** (done) | Lazy only with open paragraph / link-def title / ordered start≠1 | +| **23** (done) | Pinned reparse windows; `serializeDocument` without a whole-output parse | ## Replace boundary diff --git a/docs/design/roadmap.md b/docs/design/roadmap.md index f0303ce..79ab6e1 100644 --- a/docs/design/roadmap.md +++ b/docs/design/roadmap.md @@ -313,3 +313,19 @@ Phase 21 **shipped** on `main` (v0.1.18). Phase 22 **shipped** on `main` (v0.1.19). +## Phase 23 acceptance + +- [x] A reparse window is pinned at both edges (`src/window.ts`): it opens at the + start of an untouched block, and must close with the untouched block after + it, exactly as that block was; otherwise it reads on to the end of the text +- [x] Frontmatter, the one block decided by unbounded lookahead, is refused by a + window that opens the note and stops short of its close +- [x] `reparseBlocks` / `reparseFromText` match `parseBlocks` on generated edits, + including unclosed fences, math, comments and lazy continuation + (`reparse-window.test.ts`) +- [x] `serializeDocument` returns the document `parseDocument(outputBytes)` would, + without a whole parse, on generated saves over LF, CRLF, BOM, no final + newline and no frontmatter (`serialize-incremental.test.ts`) +- [x] Package `0.1.20` + +Phase 23 **shipped** on `main` (v0.1.20). diff --git a/package.json b/package.json index a465f04..bd42307 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@roobli/md", - "version": "0.1.19", + "version": "0.1.20", "description": "WYSIWYG-first markdown engine for Noto — block-oriented, byte-exact friendly, fast on large vaults.", "license": "MIT", "type": "module", diff --git a/src/backend/native-scanner.ts b/src/backend/native-scanner.ts index cd18e74..8b7b205 100644 --- a/src/backend/native-scanner.ts +++ b/src/backend/native-scanner.ts @@ -514,8 +514,12 @@ function spansToSplit(text: string, raw: { kind: BlockKind; start: number; end: * Native split for the Phase 1–5 dialect. Always returns a SplitDocument * (never null). Micromark is not consulted — see `@roobli/md/legacy-micromark` * for the Phase 0 compatibility backend. + * + * `atDocumentStart` is false when `text` is a window cut from the middle of a + * document: YAML frontmatter is only recognised at the very start of a file, + * so a `---` that opens a window is a thematic break there, not frontmatter. */ -export function tryNativeSplit(text: string): SplitDocument { +export function tryNativeSplit(text: string, atDocumentStart = true): SplitDocument { if (text.length === 0) { return { spans: [], leading: '', gaps: [], trailing: '' }; } @@ -532,7 +536,7 @@ export function tryNativeSplit(text: string): SplitDocument { } // YAML frontmatter (only at absolute document start) - if (i === 0 && line.start === 0 && isFrontmatterOpen(line.content, true)) { + if (atDocumentStart && i === 0 && line.start === 0 && isFrontmatterOpen(line.content, true)) { let j = i + 1; let closed = false; while (j < lines.length) { diff --git a/src/parse.ts b/src/parse.ts index 06fe815..3bdff57 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -7,16 +7,21 @@ function hasBom(bytes: Uint8Array): boolean { return bytes.length >= 3 && bytes[0] === UTF8_BOM[0] && bytes[1] === UTF8_BOM[1] && bytes[2] === UTF8_BOM[2]; } -function detectLineEnding(text: string): EngineEnvelope['lineEnding'] { - let crlf = 0; - let bareLf = 0; - for (let index = 0; index < text.length; index += 1) { - if (text[index] !== '\n') continue; - if (index > 0 && text[index - 1] === '\r') crlf += 1; - else bareLf += 1; - } - if (crlf > 0 && bareLf > 0) return 'mixed'; - return crlf > 0 ? 'crlf' : 'lf'; +/** An LF with no CR before it. No `g` flag, so the regex carries no state. */ +const BARE_LF = /(?:^|[^\r])\n/; + +/** + * `crlf` when every line ends CRLF, `lf` when none does, `mixed` otherwise. + * + * Two native scans (`includes` and one regex test) rather than a character + * loop: on an 8 MB note this was 20 to 35 ms of every parse and is now well + * under one. + */ +export function detectLineEnding(text: string): EngineEnvelope['lineEnding'] { + const crlf = text.includes('\r\n'); + const bareLf = BARE_LF.test(text); + if (crlf && bareLf) return 'mixed'; + return crlf ? 'crlf' : 'lf'; } function decodeUtf8(bytes: Uint8Array): string | null { diff --git a/src/reparse-window.test.ts b/src/reparse-window.test.ts new file mode 100644 index 0000000..70aaee3 --- /dev/null +++ b/src/reparse-window.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from 'vitest'; +import { parseBlocks } from './parse.js'; +import { reparseFromText } from './reparse.js'; +import type { SplitDocument } from './types.js'; + +/** + * An incremental reparse is only correct if it says what a whole parse of the + * same text says. These hold it to exactly that, including for edits whose + * effect reaches well past the blocks they touched. + */ + +function shape(split: SplitDocument) { + return { + spans: split.spans.map((span) => [span.kind, span.start, span.end, span.markdown]), + leading: split.leading, + gaps: split.gaps, + trailing: split.trailing, + }; +} + +function expectSameAsFull(prior: SplitDocument, next: string, neighborSlack: number, label = ''): void { + const incremental = reparseFromText(prior, next, { neighborSlack }); + expect(shape(incremental), label).toEqual(shape(parseBlocks(next))); +} + +const PARAGRAPHS = Array.from({ length: 12 }, (_, index) => `Paragraph ${index}.`).join('\n\n') + '\n'; + +describe('reparse windows that an edit runs out of', () => { + it('follows an unclosed fence to the end of the note', () => { + const prior = parseBlocks(PARAGRAPHS); + const at = PARAGRAPHS.indexOf('Paragraph 3.'); + const next = `${PARAGRAPHS.slice(0, at)}\`\`\`\n${PARAGRAPHS.slice(at)}`; + for (const slack of [0, 1, 2]) expectSameAsFull(prior, next, slack, `slack ${slack}`); + expect(parseBlocks(next).spans.at(-1)!.kind).toBe('fenced-code'); + }); + + it('follows an unclosed comment, and math, the same way', () => { + const prior = parseBlocks(PARAGRAPHS); + const at = PARAGRAPHS.indexOf('Paragraph 5.'); + for (const opener of ['', '
\n', '
\n', '- ', '* ', '1. ', '> ', '# ', + '---\n', '===\n', ' ', '\n', '\n\n', '| x | y |\n| - | - |\n', '[r]: /u\n', '[^n]: note\n', 'word ', 'text\n', +]; + +describe('reparse windows, against a whole parse, on generated edits', () => { + for (const slack of [0, 1]) { + it(`agrees with parseBlocks on 600 random edits (slack ${slack})`, () => { + const prior = parseBlocks(SOURCE); + for (let seed = 1; seed <= 600; seed += 1) { + const random = rng(seed * 7 + slack); + const at = Math.floor(random() * (SOURCE.length + 1)); + const pick = random(); + let next: string; + if (pick < 0.55) { + const snippet = SNIPPETS[Math.floor(random() * SNIPPETS.length)]!; + next = SOURCE.slice(0, at) + snippet + SOURCE.slice(at); + } else if (pick < 0.85) { + const end = Math.min(SOURCE.length, at + 1 + Math.floor(random() * 40)); + next = SOURCE.slice(0, at) + SOURCE.slice(end); + } else { + const end = Math.min(SOURCE.length, at + 1 + Math.floor(random() * 20)); + const snippet = SNIPPETS[Math.floor(random() * SNIPPETS.length)]!; + next = SOURCE.slice(0, at) + snippet + SOURCE.slice(end); + } + expectSameAsFull(prior, next, slack, `seed ${seed}`); + } + }); + } +}); diff --git a/src/reparse.ts b/src/reparse.ts index 40663fd..50b5126 100644 --- a/src/reparse.ts +++ b/src/reparse.ts @@ -12,6 +12,7 @@ */ import { joinSplit, parseBlocks } from './parse.js'; +import { parseWindow } from './window.js'; import type { BlockSpan, SplitDocument } from './types.js'; /** Character edit against the prior joined source (`joinSplit(prior)`). */ @@ -202,18 +203,6 @@ function resolveDirtyOrdinals( return { from, to }; } -function priorWindow( - prior: SplitDocument, - priorText: string, - dirtyFrom: number, - dirtyTo: number, -): { start: number; end: number } { - const n = prior.spans.length; - const start = dirtyFrom === 0 ? 0 : prior.spans[dirtyFrom - 1]!.end; - const end = dirtyTo === n - 1 ? priorText.length : prior.spans[dirtyTo + 1]!.start; - return { start, end }; -} - function shiftSpan(span: BlockSpan, delta: number): BlockSpan { if (delta === 0) return span; return { @@ -225,16 +214,6 @@ function shiftSpan(span: BlockSpan, delta: number): BlockSpan { }; } -function remapLocalSpan(span: BlockSpan, windowStart: number): BlockSpan { - return { - kind: span.kind, - start: span.start + windowStart, - end: span.end + windowStart, - markdown: span.markdown, - node: span.node, - }; -} - function stitchCoverage(text: string, spans: readonly BlockSpan[], prior: SplitDocument, dirtyFrom: number, dirtyTo: number, middleCount: number): SplitDocument { if (spans.length === 0) { return { spans: [], leading: '', gaps: [], trailing: text }; @@ -311,34 +290,47 @@ export function reparseBlocks(options: ReparseBlocksOptions): ReparseBlocksResul const dirty = resolveDirtyOrdinals(prior, edit, replacedBlocks, slack); if (dirty === 'full' || prior.spans.length === 0) return finishFull(); - const { from: dirtyFrom, to: dirtyTo } = dirty; - const window = priorWindow(prior, priorText, dirtyFrom, dirtyTo); - - let windowStart: number; - let windowEnd: number; - if (edit) { - windowStart = mapPriorOffsetExclusiveEnd(window.start, edit); - windowEnd = mapPriorOffsetExclusiveEnd(window.end, edit); - } else { - // Block-ordinal path: host guarantees the change is confined to the dirty - // window, so prefix bytes are unchanged and suffix length is preserved. - windowStart = window.start; - windowEnd = text.length - (priorText.length - window.end); - } + const n = prior.spans.length; + // Everything before the edit is where it was; everything after it moved by + // the same amount. On the block path the host vouches that the change sits + // inside the dirty ordinals. + const delta = edit + ? edit.inserted.length - (edit.priorEnd - edit.priorStart) + : text.length - priorText.length; - if (windowStart < 0 || windowEnd < windowStart || windowEnd > text.length) { + // The window is pinned at both edges (see `window.ts`): it opens at the start + // of the untouched block before the dirty ordinals, and must close with the + // untouched block after them, exactly as that block was. If the edit ran on + // into it, as an unclosed fence does, the window is read on to the end of + // the text instead. + const leftOrdinal = dirty.from - 1; + const rightOrdinal = dirty.to + 1; + const left = leftOrdinal >= 0 ? prior.spans[leftOrdinal]! : null; + const right = rightOrdinal < n ? shiftSpan(prior.spans[rightOrdinal]!, delta) : null; + const windowStart = left ? left.start : 0; + if ( + (left && text.slice(left.start, left.end) !== left.markdown) + || (right && (right.end > text.length || text.slice(right.start, right.end) !== right.markdown)) + ) { return finishFull(); } - const slice = text.slice(windowStart, windowEnd); - const local = parseBlocks(slice); - const middle = local.spans.map((span) => remapLocalSpan(span, windowStart)); + let local = right ? parseWindow(text, windowStart, right.end, right) : null; + const anchored = local !== null; + if (local === null) local = parseWindow(text, windowStart, text.length, null); + if (local === null) return finishFull(); + const windowEnd = anchored ? right!.end : text.length; + + // The block before the window keeps its identity unless the edit changed how + // it ends (a line that now continues it, say). + const leftUnchanged = left !== null && local.length > 0 + && local[0]!.start === left.start && local[0]!.end === left.end && local[0]!.kind === left.kind; + const dirtyFrom = leftUnchanged ? leftOrdinal + 1 : Math.max(0, leftOrdinal); + const dirtyTo = anchored ? rightOrdinal - 1 : n - 1; + const middle = local.slice(leftUnchanged ? 1 : 0, anchored ? -1 : undefined); const prefix = prior.spans.slice(0, dirtyFrom) as BlockSpan[]; - const delta = edit - ? edit.inserted.length - (edit.priorEnd - edit.priorStart) - : text.length - priorText.length; - const suffix = prior.spans.slice(dirtyTo + 1).map((span) => shiftSpan(span, delta)); + const suffix = anchored ? prior.spans.slice(rightOrdinal).map((span) => shiftSpan(span, delta)) : []; const spans: BlockSpan[] = [...prefix, ...middle, ...suffix]; const stitched = stitchCoverage(text, spans, prior, dirtyFrom, dirtyTo, middle.length); diff --git a/src/serialize-incremental.test.ts b/src/serialize-incremental.test.ts new file mode 100644 index 0000000..4f46765 --- /dev/null +++ b/src/serialize-incremental.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from 'vitest'; +import { parseDocument } from './parse.js'; +import { identityUnits, serializeDocument, type SerializeUnit } from './serialize.js'; +import type { EngineDocument } from './types.js'; + +/** + * A save builds the next document from the blocks it moved and a few small + * reparses instead of parsing its whole output again. That is only sound if + * the document it builds is the one a parse of the output would give; these + * hold it to exactly that. + */ + +const encoder = new TextEncoder(); + +function parsed(text: string, bom = false): EngineDocument { + const body = encoder.encode(text); + const bytes = bom ? Uint8Array.from([0xef, 0xbb, 0xbf, ...body]) : body; + const result = parseDocument(bytes); + if (result.status !== 'parsed') throw new Error(result.message); + return result.document; +} + +function shape(document: EngineDocument) { + return { + envelope: document.envelope, + text: document.text, + leading: document.leading, + trailing: document.trailing, + gaps: document.gaps, + blocks: document.blocks.map((block) => [block.kind, block.start, block.end, block.markdown, block.node]), + }; +} + +/** Serialize, then check the document against a parse of the output. */ +function expectFaithful(document: EngineDocument, units: SerializeUnit[], label: string, envelope?: { hasFinalNewline?: boolean }) { + const result = envelope ? serializeDocument(document, { units, envelope }) : serializeDocument(document, { units }); + if (result.status !== 'serialized') return result; + const full = parseDocument(result.outputBytes); + if (full.status !== 'parsed') throw new Error(full.message); + expect(shape(result.document), label).toEqual(shape(full.document)); + return result; +} + +const SOURCE = `--- +title: Saves +--- + +# Title + +A paragraph with *emphasis* and a [link](https://example.com). +It continues on a second line. + +- first item +- second item + - nested item + +Between the lists. + +- another list + +1. one +2. two + +> A quote +> that goes on. + +\`\`\`ts +const value = 1; + +const other = 2; +\`\`\` + +| a | b | +| - | - | +| 1 | 2 | + +$$ +x = y^2 +$$ + +
html
+ +Text with a footnote.[^f] + +[^f]: The footnote. + +[ref]: https://example.com/ref + +Setext heading +-------------- + + indented code + +*** + +Last paragraph. +`; + +const REPLACEMENTS = [ + 'A plain paragraph.', '```\nnever closed', '$$\nnever closed', '