diff --git a/CHANGELOG.md b/CHANGELOG.md index 461a161..cd78ba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ change the API; each entry says so when it does. Phases refer to [`docs/design/roadmap.md`](docs/design/roadmap.md). +## [0.1.20] - 2026-09-30 + +### Fixed +- `reparseBlocks` and `reparseFromText` no longer cut a block short at the + edge of their window. An edit that opened a code fence, a display-math block + or an HTML comment without closing it was read as one short block followed + by the old blocks after it, where a whole parse has it run to the end of the + text. The window now has to end with the untouched block after it, exactly + as that block was; when it does not, the reparse reads on to the end. A line + that now continues the block before the window (lazy continuation) is also + picked up. (Phase 23.) + +### Changed +- `serializeDocument` builds the next document from the blocks it moved and + small reparses around what changed, pinned the same way, instead of parsing + its whole output again. The document is the one `parseDocument` would + return, which the tests check on thousands of generated saves; the whole + parse remains as the fallback. On an 8 MB note a one-block save goes from + about 130-200 ms to 40-80 ms. (Phase 23.) +- Line-ending detection uses two native scans instead of a character loop, + which took 20-35 ms of every parse of an 8 MB note. + ## [0.1.19] - 2026-09-29 ### Fixed @@ -140,6 +162,7 @@ Phases refer to [`docs/design/roadmap.md`](docs/design/roadmap.md). `@roobli/md/legacy-micromark`. (Phase 6; breaking for callers that relied on the fallback.) +[0.1.20]: https://github.com/roobli/md/compare/v0.1.19...v0.1.20 [0.1.19]: https://github.com/roobli/md/compare/v0.1.18...v0.1.19 [0.1.18]: https://github.com/roobli/md/compare/v0.1.17...v0.1.18 [0.1.17]: https://github.com/roobli/md/compare/v0.1.16...v0.1.17 diff --git a/README.md b/README.md index 5df7557..1bf883d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,10 @@ AGPL. - **Stable spans.** Every block carries its `start` and `end` in the source, its kind, and the literal gap before it. - **Local reparse.** `reparseBlocks` and `reparseFromText` rebuild only the - window around an edit; blocks outside it keep their identity. + window around an edit; blocks outside it keep their identity. The window is + pinned at both edges, so the result is always what a whole parse would say; + an edit that runs past it (an unclosed fence) is read on to the end. + `serializeDocument` builds its next document the same way. - **One dialect.** CommonMark plus GFM tables, task lists and strikethrough, `$`/`$$` math, YAML frontmatter, footnotes, callouts, wiki links and CJK-friendly emphasis. Behaviour is checked against micromark for parity. @@ -45,7 +48,7 @@ Design notes: [vision](docs/design/vision.md) · ## Install ``` -pnpm add github:roobli/md#v0.1.19 +pnpm add github:roobli/md#v0.1.20 ``` Pin a tag: `main` moves. A host that runs install scripts may need to allow diff --git a/docs/design/contract-v0.md b/docs/design/contract-v0.md index b5d3fca..98e3ea2 100644 --- a/docs/design/contract-v0.md +++ b/docs/design/contract-v0.md @@ -84,11 +84,19 @@ only the dirty window (widened by `neighborSlack`), and stitches prefix / middle / suffix with absolute offsets. Untouched prefix spans keep object identity; untouched suffix spans keep `markdown` string identity. Micromark is not imported on this path (Phase 6); use `@roobli/md/legacy-micromark` for compat. +The window is pinned at both edges (Phase 23, `src/window.ts`). It opens at the +start of the untouched block before the dirty ordinals, which is re-read, so a +line that now continues that block is picked up. It must close with the +untouched block after them, exactly as that block was; if the edit ran on into +it (an unclosed fence, display math or comment), the reparse reads on to the +end of the text instead. The result is always what `parseBlocks` of the whole +text would give; `neighborSlack` only trades window size for how often the +fallback is needed, and is no longer what keeps the result correct. + Noto call sites: `replaceMarkdown` can pass the differing middle ordinals as `replacedBlocks` instead of `splitBlocks(fullMarkdown)`; when the host only has the full next buffer, prefer `reparseFromText(prior, nextText)` (Phase 11). -The single-block save path can use `neighborSlack: 0` after `parseSingleBlock` -validation. +`neighborSlack: 0` is safe on every path. ### Serialize (Phase 5) @@ -224,6 +232,7 @@ Full table: [`roadmap.md`](./roadmap.md). Contract-facing summary: | **20** (done) | Empty list item + blank → structural outside | | **21** (done) | List-steal pipe-less delimiters not tables | | **22** (done) | Lazy only with open paragraph / link-def title / ordered start≠1 | +| **23** (done) | Pinned reparse windows; `serializeDocument` without a whole-output parse | ## Replace boundary diff --git a/docs/design/roadmap.md b/docs/design/roadmap.md index f0303ce..79ab6e1 100644 --- a/docs/design/roadmap.md +++ b/docs/design/roadmap.md @@ -313,3 +313,19 @@ Phase 21 **shipped** on `main` (v0.1.18). Phase 22 **shipped** on `main` (v0.1.19). +## Phase 23 acceptance + +- [x] A reparse window is pinned at both edges (`src/window.ts`): it opens at the + start of an untouched block, and must close with the untouched block after + it, exactly as that block was; otherwise it reads on to the end of the text +- [x] Frontmatter, the one block decided by unbounded lookahead, is refused by a + window that opens the note and stops short of its close +- [x] `reparseBlocks` / `reparseFromText` match `parseBlocks` on generated edits, + including unclosed fences, math, comments and lazy continuation + (`reparse-window.test.ts`) +- [x] `serializeDocument` returns the document `parseDocument(outputBytes)` would, + without a whole parse, on generated saves over LF, CRLF, BOM, no final + newline and no frontmatter (`serialize-incremental.test.ts`) +- [x] Package `0.1.20` + +Phase 23 **shipped** on `main` (v0.1.20). diff --git a/package.json b/package.json index a465f04..bd42307 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@roobli/md", - "version": "0.1.19", + "version": "0.1.20", "description": "WYSIWYG-first markdown engine for Noto — block-oriented, byte-exact friendly, fast on large vaults.", "license": "MIT", "type": "module", diff --git a/src/backend/native-scanner.ts b/src/backend/native-scanner.ts index cd18e74..8b7b205 100644 --- a/src/backend/native-scanner.ts +++ b/src/backend/native-scanner.ts @@ -514,8 +514,12 @@ function spansToSplit(text: string, raw: { kind: BlockKind; start: number; end: * Native split for the Phase 1–5 dialect. Always returns a SplitDocument * (never null). Micromark is not consulted — see `@roobli/md/legacy-micromark` * for the Phase 0 compatibility backend. + * + * `atDocumentStart` is false when `text` is a window cut from the middle of a + * document: YAML frontmatter is only recognised at the very start of a file, + * so a `---` that opens a window is a thematic break there, not frontmatter. */ -export function tryNativeSplit(text: string): SplitDocument { +export function tryNativeSplit(text: string, atDocumentStart = true): SplitDocument { if (text.length === 0) { return { spans: [], leading: '', gaps: [], trailing: '' }; } @@ -532,7 +536,7 @@ export function tryNativeSplit(text: string): SplitDocument { } // YAML frontmatter (only at absolute document start) - if (i === 0 && line.start === 0 && isFrontmatterOpen(line.content, true)) { + if (atDocumentStart && i === 0 && line.start === 0 && isFrontmatterOpen(line.content, true)) { let j = i + 1; let closed = false; while (j < lines.length) { diff --git a/src/parse.ts b/src/parse.ts index 06fe815..3bdff57 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -7,16 +7,21 @@ function hasBom(bytes: Uint8Array): boolean { return bytes.length >= 3 && bytes[0] === UTF8_BOM[0] && bytes[1] === UTF8_BOM[1] && bytes[2] === UTF8_BOM[2]; } -function detectLineEnding(text: string): EngineEnvelope['lineEnding'] { - let crlf = 0; - let bareLf = 0; - for (let index = 0; index < text.length; index += 1) { - if (text[index] !== '\n') continue; - if (index > 0 && text[index - 1] === '\r') crlf += 1; - else bareLf += 1; - } - if (crlf > 0 && bareLf > 0) return 'mixed'; - return crlf > 0 ? 'crlf' : 'lf'; +/** An LF with no CR before it. No `g` flag, so the regex carries no state. */ +const BARE_LF = /(?:^|[^\r])\n/; + +/** + * `crlf` when every line ends CRLF, `lf` when none does, `mixed` otherwise. + * + * Two native scans (`includes` and one regex test) rather than a character + * loop: on an 8 MB note this was 20 to 35 ms of every parse and is now well + * under one. + */ +export function detectLineEnding(text: string): EngineEnvelope['lineEnding'] { + const crlf = text.includes('\r\n'); + const bareLf = BARE_LF.test(text); + if (crlf && bareLf) return 'mixed'; + return crlf ? 'crlf' : 'lf'; } function decodeUtf8(bytes: Uint8Array): string | null { diff --git a/src/reparse-window.test.ts b/src/reparse-window.test.ts new file mode 100644 index 0000000..70aaee3 --- /dev/null +++ b/src/reparse-window.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from 'vitest'; +import { parseBlocks } from './parse.js'; +import { reparseFromText } from './reparse.js'; +import type { SplitDocument } from './types.js'; + +/** + * An incremental reparse is only correct if it says what a whole parse of the + * same text says. These hold it to exactly that, including for edits whose + * effect reaches well past the blocks they touched. + */ + +function shape(split: SplitDocument) { + return { + spans: split.spans.map((span) => [span.kind, span.start, span.end, span.markdown]), + leading: split.leading, + gaps: split.gaps, + trailing: split.trailing, + }; +} + +function expectSameAsFull(prior: SplitDocument, next: string, neighborSlack: number, label = ''): void { + const incremental = reparseFromText(prior, next, { neighborSlack }); + expect(shape(incremental), label).toEqual(shape(parseBlocks(next))); +} + +const PARAGRAPHS = Array.from({ length: 12 }, (_, index) => `Paragraph ${index}.`).join('\n\n') + '\n'; + +describe('reparse windows that an edit runs out of', () => { + it('follows an unclosed fence to the end of the note', () => { + const prior = parseBlocks(PARAGRAPHS); + const at = PARAGRAPHS.indexOf('Paragraph 3.'); + const next = `${PARAGRAPHS.slice(0, at)}\`\`\`\n${PARAGRAPHS.slice(at)}`; + for (const slack of [0, 1, 2]) expectSameAsFull(prior, next, slack, `slack ${slack}`); + expect(parseBlocks(next).spans.at(-1)!.kind).toBe('fenced-code'); + }); + + it('follows an unclosed comment, and math, the same way', () => { + const prior = parseBlocks(PARAGRAPHS); + const at = PARAGRAPHS.indexOf('Paragraph 5.'); + for (const opener of ['', '