From 15d0e73ee1e391f7928c2ace913c69f7ef5e1e88 Mon Sep 17 00:00:00 2001 From: Daniel X Moore Date: Sat, 5 Sep 2026 08:25:40 -0700 Subject: [PATCH] Simplify compiler internals; fix event leakage in typed parsers Refactor (byte-identical output): - Merge the three terminal-interning functions into one `intern` helper. - Drop the `types` parameter threaded through `compileOp`; it was never read. - Write `tsHead` as a literal instead of deriving it from `mjsHead` with a regex replace and a match on the import block. - Emit the braced rule-name block from one `ruleNameBlock` helper, used by both the grammar object and the ESM export block. - Collapse `prettyPrint` to a single expression. Fix: - The typed parser tail hoisted the Validator and context out of `parse` into an IIFE, so `Object.assign(ctx, options.events)` from one call persisted into the next. The untyped tail created them per call. Emit one tail for both, with type annotations slotted in when `types` is set, using the per-call structure. Typed output changes only in the tail. Adds a regression test for both JS and typed parsers. Verified against a snapshot of compiler output for source/hera.hera, every sample, and the inference fixture across ten option combinations. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P6RAoyCaAv1Xe7FjMYwD3b --- CHANGELOG.md | 3 + source/compiler.civet | 285 ++++++++++++++++++++---------------------- test/main.civet | 19 ++- 3 files changed, 155 insertions(+), 152 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02e8126..dbec8b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ All notable changes to `@danielx/hera` will be documented in this file. from ~990 KB to ~77 KB. Generated source maps are unchanged. ### Fixed +- Parsers compiled with `types: true` no longer carry `events` from one + `parse()` call over into later calls. The typed and untyped outputs now + share the same per-call parser tail. - Type declarations for `$C` and `$S` now support generated parsers with more than 9 choice alternatives or sequence items. diff --git a/source/compiler.civet b/source/compiler.civet index 9dd3e0a..7f17e98 100644 --- a/source/compiler.civet +++ b/source/compiler.civet @@ -26,46 +26,26 @@ reDefs: string[] := [] dynamicReDefs: RegExpPart[][] := [] /** -Define a literal string terminal +Return the `${prefix}${index}` identifier for `value`, adding it to `defs` on +first sight so identical terminals share a single definition. `key` maps a +value to the identity used for de-duplication. */ -defineTerminal := (lit: string) -> - index := strDefs.indexOf(lit) - let id +intern := (defs: T[], prefix: string, value: T, key: (v: T) => unknown = (v) => v): string -> + k := key value + index .= defs.findIndex (entry) -> key(entry) is k + if index < 0 + index = defs.length + defs.push value + `${prefix}${index}` - if index >= 0 - id = `$L${index}` - else - id = `$L${strDefs.length}` - strDefs.push lit - - return id - -/** -Define a RegExp terminal -*/ -defineRe := (re: string) -> - index := reDefs.indexOf(re) - let id +/** Define a literal string terminal */ +defineTerminal := (lit: string) -> intern strDefs, "$L", lit - if index >= 0 - id = `$R${index}` - else - id = `$R${reDefs.length}` - reDefs.push re - - return id - -defineDynamicRe := (parts: RegExpPart[]) => - index := dynamicReDefs.findIndex((entry) => JSON.stringify(entry) is JSON.stringify(parts)) - let id - - if index >= 0 - id = `$RD${index}` - else - id = `$RD${dynamicReDefs#}` - dynamicReDefs.push parts +/** Define a RegExp terminal */ +defineRe := (re: string) -> intern reDefs, "$R", re - return id +/** Define an interpolated (dynamic) RegExp terminal */ +defineDynamicRe := (parts: RegExpPart[]) -> intern dynamicReDefs, "$RD", parts, JSON.stringify isIdentifierExpression := (expression: string): boolean => /^[$_\p{ID_Start}][$\u200C\u200D\p{ID_Continue}]*$/u.test(expression.trim()) @@ -90,18 +70,12 @@ heregexToExpression := (parts: RegExpPart[], constantPrefix?: string): string => Pretty print a string or RegExp literal */ prettyPrint := (name: string, terminal: string, re?: boolean) -> - let pv - if re - pv = `/${terminal}/` - else - pv = JSON.stringify(terminal) - - return `${name} ${pv}` + `${name} ${re ? `/${terminal}/` : JSON.stringify(terminal)}` /** * Compile an operator to a JS or TS string. */ -compileOp := (tuple: HeraAST, name: string, defaultHandler: boolean, types?: boolean): string -> +compileOp := (tuple: HeraAST, name: string, defaultHandler: boolean): string -> // TODO: should nested levels have default handler set to true? (only comes into play on regexps) if Array.isArray(tuple) switch tuple[0] @@ -132,7 +106,7 @@ compileOp := (tuple: HeraAST, name: string, defaultHandler: boolean, types?: boo when "/" { src := tuple[1].map (arg) -> - compileOp(arg, name, defaultHandler, types) + compileOp(arg, name, defaultHandler) .join(", ") `$C(${src})` @@ -140,26 +114,26 @@ compileOp := (tuple: HeraAST, name: string, defaultHandler: boolean, types?: boo when "S" { src := tuple[1].map (arg) -> - compileOp(arg, name, defaultHandler, types) + compileOp(arg, name, defaultHandler) .join(", ") `$S(${src})` } when "*" - `$Q(${compileOp(tuple[1], name, defaultHandler, types)})` + `$Q(${compileOp(tuple[1], name, defaultHandler)})` when "+" - `$P(${compileOp(tuple[1], name, defaultHandler, types)})` + `$P(${compileOp(tuple[1], name, defaultHandler)})` when "?" - `$E(${compileOp(tuple[1], name, defaultHandler, types)})` + `$E(${compileOp(tuple[1], name, defaultHandler)})` when "$" // Inside text can ignore all handlers since they are disregarded anyway - `$TEXT(${compileOp(tuple[1], name, false, types)})` + `$TEXT(${compileOp(tuple[1], name, false)})` when "&" - `$Y(${compileOp(tuple[1], name, defaultHandler, types)})` + `$Y(${compileOp(tuple[1], name, defaultHandler)})` when "!" - `$N(${compileOp(tuple[1], name, defaultHandler, types)})` + `$N(${compileOp(tuple[1], name, defaultHandler)})` default if tuple[0].name - compileOp(tuple[1], name, defaultHandler, types) + compileOp(tuple[1], name, defaultHandler) else throw new Error `Unknown op: ${tuple[0]} ${JSON.stringify(tuple[1])}` else // rule reference @@ -372,7 +346,7 @@ compileRuleBodyInline := (options: CompilerOptions, name: string, fnName: string // if the length is 1, like the no-handler case. if typeOnlyHandler and rule[0] is "S" and rule[1]# is 1 rule = rule[1][0] - parserExpr := compileOp(rule, name, true, types) + parserExpr := compileOp(rule, name, true) helpers.push `const ${fnName}$parser = ${parserExpr};` body := if wrapEvent [ @@ -394,7 +368,7 @@ compileRuleBodyInline := (options: CompilerOptions, name: string, fnName: string unless h is like { f } and h.f `\n}` ] -compileRulesObject := (ruleNames: string[]) -> - meat := ruleNames.map (name) -> - ` ${name}` - .join(",\n") - - return `{\n${meat}}` +/** +Braced block with one rule name per line, indented, comma separated. +`closeOnOwnLine` puts the closing brace on its own line. +*/ +ruleNameBlock := (ruleNames: string[], closeOnOwnLine = false) -> + names := ruleNames.map((name) -> ` ${name}`).join(",\n") + `{\n${names}${closeOnOwnLine ? "\n" : ""}}` compileExports := (ruleNames: string[], module: boolean) -> if module - meat := ruleNames.map (name) -> - ` ${name}` - .join(",\n") - - return `export {\n${meat}\n}` - - return ruleNames.map (name) -> - `exports.${name} = ${name};` - .join("\n") + `export ${ruleNameBlock ruleNames, true}` + else + ruleNames.map((name) -> `exports.${name} = ${name};`).join("\n") /** Get the named handler parameter, if any, from an operator wrapper. @@ -720,7 +689,7 @@ export function compile(rules: HeraRules, maybeOptions?: CompilerOptions): strin else if module then mjsHead else cjsHead .replace("\"./machine.js\"", JSON.stringify(libPath)) - tail := types ? tsTail : jsTail + tail := parserTail types exp := (module ? mjsExport : cjsExport) // User code blocks (the ``` blocks in a .hera file) are appended verbatim @@ -733,7 +702,7 @@ export function compile(rules: HeraRules, maybeOptions?: CompilerOptions): strin rules[Symbol.for("code")] code := generate [ head - `\n\nconst grammar = ${compileRulesObject(ruleNames)};\n\n` + `\n\nconst grammar = ${ruleNameBlock ruleNames};\n\n` `\n\nconst grammarDefaultRule = ${JSON.stringify(ruleNames[0])};\n\n` `const $skip${types ? ": never" : ""} = SKIP${types ? " as never" : ""}; void $skip;\n\n` strDefSource @@ -833,36 +802,51 @@ cjsHead := """ """ -jsTail := """ - const parser = { - parse: (input, options = {}) => { - const { fail, validate, reset } = Validator() - let ctx = { expectation: "", fail } - if (typeof input !== "string") throw new Error("Input must be a string") - - let parser - if (options.startRule !== null && options.startRule !== undefined) - parser = grammar[options.startRule] - else - parser = Object.values(grammar)[0] - - if (!parser) throw new Error(`Could not find rule with name '${options.startRule}'`) - - const filename = options.filename || ""; - - reset() - Object.assign(ctx, { ...options.events }); - - return validate(input, parser(ctx, { - input, - pos: 0, - }), { - filename: filename - }) - } - } -""" +// The generated `parser` object. A fresh Validator and context are created +// on every parse call so `events` from one call never leak into the next. +// Type annotations are added when `types` is set; the value structure is +// otherwise identical. +parserTail := (types: boolean): string -> + t := (annotation: string) -> types ? annotation : "" + cast := t " as Parser>" + typeDefs: string[] := types ? [ + "type Grammar = typeof grammar;" + "type GrammarDefaultRule = typeof grammarDefaultRule;" + "type ParserResult

= P extends Parser ? T : never;" + "" + ] : [] + [ + ...typeDefs + "const parser = {" + ` parse: ${t ""}(input${t ": string"}, options${t ": ParserOptions & { startRule?: K }"} = {}) => {` + " const { fail, validate, reset } = Validator()" + ` let ctx${t ": ParserContext"} = { expectation: "", fail }` + "" + ' if (typeof input !== "string") throw new Error("Input must be a string")' + "" + " let parser" + " if (options.startRule !== null && options.startRule !== undefined)" + ` parser = grammar[options.startRule]${cast}` + " else" + ` parser = Object.values(grammar)[0]${cast}` + "" + ' if (!parser) throw new Error(`Could not find rule with name \'${options.startRule}\'`)' + "" + ' const filename = options.filename || "";' + "" + " reset()" + " Object.assign(ctx, { ...options.events });" + "" + " return validate(input, parser(ctx, {" + " input," + " pos: 0," + " }), {" + " filename: filename" + " })" + " }" + "}" + ].join "\n" cjsExport := """ exports.default = parser @@ -891,60 +875,59 @@ mjsHead := """ """ -tsHead := mjsHead.replace("}", """ - type Loc, - type MaybeResult, - type ParseResult, - type Parser, - type ParserContext, - type ParserOptions, - type ParseState, - type Unwrap, -} -""") + - "\nvoid " + mjsHead.match(/\{[\s\S]+\}/m)![0] + // quick-and-dirty way to void all the imports so that TS doesn't complain if they aren't used - "\n// Reference all imported types at value-position so TS doesn't flag them as unused.\nconst _types: Loc | MaybeResult | ParseResult | Parser | ParserContext | ParserOptions | ParseState | Unwrap> | undefined = undefined; void _types;\n" - - -tsTail := """ - type Grammar = typeof grammar; - type GrammarDefaultRule = typeof grammarDefaultRule; - type ParserResult

= P extends Parser ? T : never; - - const parser = (function() { - const { fail, validate, reset } = Validator() - let ctx: ParserContext = { expectation: "", fail } - - return { - parse: ( - input: string, - options: ParserOptions & { startRule?: K } = {} - ) => { - if (typeof input !== "string") throw new Error("Input must be a string") - - let parser - if (options.startRule !== null && options.startRule !== undefined) - parser = grammar[options.startRule] as Parser> - else - parser = Object.values(grammar)[0] as Parser> +// Same as mjsHead plus the runtime types. The `void {...}` block and the +// `_types` reference keep TS from flagging any of the imports as unused in +// generated parsers that don't happen to use them. +tsHead := """ + import { + $C, + $E, + $EXPECT, + $L, + $N, + $P, + $Q, + $R, + $R$0, + $S, + $TEXT, + $Y, + ParseError, + SKIP, + Validator, + type Loc, + type MaybeResult, + type ParseResult, + type Parser, + type ParserContext, + type ParserOptions, + type ParseState, + type Unwrap, + } from "./machine.js" - if (!parser) throw new Error(`Could not find rule with name '${options.startRule}'`) + void { + $C, + $E, + $EXPECT, + $L, + $N, + $P, + $Q, + $R, + $R$0, + $S, + $TEXT, + $Y, + ParseError, + SKIP, + Validator, + } + // Reference all imported types at value-position so TS doesn't flag them as unused. + const _types: Loc | MaybeResult | ParseResult | Parser | ParserContext | ParserOptions | ParseState | Unwrap> | undefined = undefined; void _types; - const filename = options.filename || ""; +""" - reset() - Object.assign(ctx, { ...options.events }); - return validate(input, parser(ctx, { - input, - pos: 0, - }), { - filename: filename - }) - } - } - }()) -""" mjsExport := """ export default parser diff --git a/test/main.civet b/test/main.civet index a5f1577..9de8078 100644 --- a/test/main.civet +++ b/test/main.civet @@ -1,6 +1,7 @@ assert from assert +typescript from typescript { readFile } from ./helper.civet -hera, { compile, generate, modImport, parse, parseTokens } from ../source/main.civet +hera, { compile, execMod, generate, modImport, parse, parseTokens } from ../source/main.civet { transpileTsToJs } from ../source/register/tsc/transpile.civet describe "Hera", -> @@ -1481,6 +1482,22 @@ describe "Hera", -> assert.deepEqual parse("abb"), ["bb", "a"] describe "events", -> + it "does not carry events over to a later parse call", -> + grammar := """ + Rule + "a" + """ + typedCode := compile grammar, types: true + parsers := [ + generate grammar + execMod transpileTsToJs typedCode, module: typescript.ModuleKind.CommonJS + ] + for { parse } of parsers + calls .= 0 + parse "a", events: { enter: -> calls++; undefined } + parse "a" + assert.equal calls, 1 + it "should pass data from enter to exit events", -> {parse} := generate """ Rule