From f14f17ee5cd123d392037175944e618d77541ba8 Mon Sep 17 00:00:00 2001 From: hellovai Date: Sun, 12 Jul 2026 00:12:33 -0700 Subject: [PATCH 1/2] feat(grammar): publish BAML TextMate grammar via self-updating mirror pipeline The grammar in typescript2/pkg-grammar now feeds every external consumer through the BoundaryML/textMate-baml mirror, with no manual steps: - build.ts also emits dist/index.js + dist/index.d.ts with the grammar inlined as a JS object literal (Shiki LanguageRegistration), so npm consumers never import JSON (Metro can't parse import attributes) - language-configuration.json moves into pkg-grammar as the source of truth; sync.mjs mirrors it into app-vscode-ext like the grammar - scripts/assemble-mirror.mjs assembles the complete mirror repo layout (npm package @boundaryml/baml-grammar + Linguist vendor dir + the mirror's own npm publish workflow) from the templates in mirror/ - sync-grammar-mirror workflow pushes the assembled mirror to BoundaryML/textMate-baml on grammar changes on canary, stamps a patch-bumped version only when content actually changed, and fails loudly if the npm release doesn't appear - new linguist-compile-gate CI job runs GitHub Linguist's own grammar-compiler (Oniguruma->PCRE conversion) against the freshly built grammar, so a change that would break .baml highlighting on github.com fails the PR instead of Linguist's next quarterly release - new tests: external-consumer invariants (scopeName/fileTypes/ self-containedness), dist-vs-raw drift, strict-JSON language config, and a whole-repo corpus tokenization pass (1293 files) that catches crashes and catastrophic-backtracking regexes - grammar-tests CI job now rebuilds and fails if committed artifacts drift from src/ Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yaml | 74 ++++++++- .github/workflows/sync-grammar-mirror.yml | 146 ++++++++++++++++++ .pre-commit-config.yaml | 2 +- .../language-configuration.json | 4 - typescript2/pkg-grammar/README.md | 49 ++++-- .../pkg-grammar/language-configuration.json | 34 ++++ typescript2/pkg-grammar/mirror/README.md | 56 +++++++ typescript2/pkg-grammar/mirror/package.json | 40 +++++ typescript2/pkg-grammar/mirror/publish.yml | 81 ++++++++++ typescript2/pkg-grammar/package.json | 13 +- .../pkg-grammar/scripts/assemble-mirror.mjs | 72 +++++++++ typescript2/pkg-grammar/scripts/build.ts | 49 +++++- typescript2/pkg-grammar/scripts/sync.mjs | 21 ++- .../tests/package-artifacts.test.ts | 105 +++++++++++++ .../pkg-grammar/tests/repo-corpus.test.ts | 63 ++++++++ typescript2/pnpm-lock.yaml | 3 + 16 files changed, 784 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/sync-grammar-mirror.yml create mode 100644 typescript2/pkg-grammar/language-configuration.json create mode 100644 typescript2/pkg-grammar/mirror/README.md create mode 100644 typescript2/pkg-grammar/mirror/package.json create mode 100644 typescript2/pkg-grammar/mirror/publish.yml create mode 100644 typescript2/pkg-grammar/scripts/assemble-mirror.mjs create mode 100644 typescript2/pkg-grammar/tests/package-artifacts.test.ts create mode 100644 typescript2/pkg-grammar/tests/repo-corpus.test.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f558dfce94..7a49d31f63a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -219,6 +219,7 @@ jobs: if git diff --quiet "${MERGE_BASE}...HEAD" -- \ ':typescript2/pkg-grammar/**' \ ':typescript2/app-vscode-ext/syntaxes/**' \ + ':typescript2/app-vscode-ext/language-configuration.json' \ ':typescript2/app-vscode-ext/package.json' \ ':typescript2/pnpm-lock.yaml' \ ':.github/actions/setup-node2/**' \ @@ -502,10 +503,79 @@ jobs: - name: "Setup Node.js for typescript2" uses: ./.github/actions/setup-node2 - - name: "Run grammar snapshot tests" + - name: "Build grammar artifacts" + run: pnpm --filter @b/pkg-grammar build + working-directory: typescript2 + + - name: "Committed artifacts must match src/" + # build regenerates baml.tmLanguage.json (+ the app-vscode-ext copies + # via sync.mjs); any diff means someone edited src/ without committing + # the regenerated artifacts, or edited an artifact by hand. + run: git diff --exit-code -- typescript2/pkg-grammar typescript2/app-vscode-ext + + - name: "Run grammar tests" run: pnpm --filter @b/pkg-grammar test working-directory: typescript2 + linguist-compile-gate: + name: "Linguist grammar-compiler gate" + runs-on: ubuntu-latest + needs: determine_changes + if: needs.determine_changes.outputs.grammar == 'true' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/canary' + steps: + - name: "Checkout Branch" + uses: useblacksmith/checkout@v1 + with: + persist-credentials: false + + - name: "Setup Node.js for typescript2" + uses: ./.github/actions/setup-node2 + + - name: "Build grammar artifacts" + run: pnpm --filter @b/pkg-grammar build + working-directory: typescript2 + + - name: "Checkout github-linguist/linguist" + env: + # GitHub's release process compiles every vendored grammar with this + # tool; running the same compile here means a grammar that passes CI + # cannot break .baml highlighting on github.com when Linguist next + # pulls the BoundaryML/textMate-baml mirror. Bump the ref alongside + # Linguist releases. + LINGUIST_REF: v9.6.0 + run: | + git clone --depth 1 --branch "$LINGUIST_REF" \ + https://github.com/github-linguist/linguist.git /tmp/linguist + + - name: "Stage candidate grammar as the textMate-baml vendor dir" + run: | + set -euo pipefail + # Drop the registered entry for our vendor dir so the compiler treats + # the staged grammar as a fresh add (avoids duplicate-scope errors + # against the currently-vendored copy). + python3 - <<'EOF' + import re + path = "/tmp/linguist/grammars.yml" + with open(path) as f: + content = f.read() + content = re.sub(r"vendor/grammars/textMate-baml:\n(- [^\n]+\n)+", "", content) + with open(path, "w") as f: + f.write(content) + EOF + rm -rf /tmp/linguist/vendor/grammars/textMate-baml + mkdir -p /tmp/linguist/vendor/grammars/textMate-baml/grammars + cp typescript2/pkg-grammar/baml.tmLanguage.json \ + /tmp/linguist/vendor/grammars/textMate-baml/grammars/ + + - name: "Compile with Linguist's grammar-compiler" + working-directory: /tmp/linguist + run: | + set -euo pipefail + export USER="${USER:-$(whoami)}" + # No -f: any regex that Linguist's Oniguruma->PCRE conversion cannot + # handle must fail this job, not surface at Linguist's next release. + script/grammar-compiler add vendor/grammars/textMate-baml + miri-tests: name: "Miri (unsafe code verification)" runs-on: blacksmith-4vcpu-ubuntu-2404 @@ -958,6 +1028,7 @@ jobs: - docs - webview-tests - grammar-tests + - linguist-compile-gate # - miri-tests # TEMPORARILY DISABLED: miri tests timing out - prof-concurrency - proto-sync @@ -984,6 +1055,7 @@ jobs: echo "| docs | ${{ needs.docs.result }} |" >> $GITHUB_STEP_SUMMARY echo "| webview-tests | ${{ needs.webview-tests.result }} |" >> $GITHUB_STEP_SUMMARY echo "| grammar-tests | ${{ needs.grammar-tests.result }} |" >> $GITHUB_STEP_SUMMARY + echo "| linguist-compile-gate | ${{ needs.linguist-compile-gate.result }} |" >> $GITHUB_STEP_SUMMARY # echo "| miri-tests | ${{ needs.miri-tests.result }} |" >> $GITHUB_STEP_SUMMARY # TEMPORARILY DISABLED echo "| prof-concurrency | ${{ needs.prof-concurrency.result }} |" >> $GITHUB_STEP_SUMMARY echo "| proto-sync | ${{ needs.proto-sync.result }} |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/sync-grammar-mirror.yml b/.github/workflows/sync-grammar-mirror.yml new file mode 100644 index 00000000000..99af55d6276 --- /dev/null +++ b/.github/workflows/sync-grammar-mirror.yml @@ -0,0 +1,146 @@ +# Sync the BAML TextMate grammar to the BoundaryML/textMate-baml mirror repo. +# +# The mirror is the single external artifact for every third-party consumer of +# the grammar: it is the npm package @boundaryml/baml-grammar (published by the +# mirror's own publish.yml, which this sync also ships), the repo GitHub +# Linguist vendors for .baml highlighting on github.com, and the source URL the +# Shiki grammar registry fetches from. Grammar edits land in +# typescript2/pkg-grammar; this workflow propagates them, so the mirror must +# never be edited by hand. +# +# Auth: the TEXTMATE_BAML_DEPLOY_KEY secret holds an SSH private key whose +# public half is a write-access deploy key on BoundaryML/textMate-baml. +# (A deploy key, not a PAT, so pushes may update the mirror's workflow files.) +# +# Bootstrap note: npm OIDC trusted publishing cannot create a brand-new +# package. The first-ever npm publish must be done once by hand from a mirror +# checkout, then the trusted publisher (repo BoundaryML/textMate-baml, workflow +# publish.yml) configured in the npm package settings. Until then the final +# verify step here fails; that is expected and is the reminder to finish setup. + +name: Sync grammar mirror + +on: + push: + branches: [canary] + paths: + - "typescript2/pkg-grammar/**" + - ".github/actions/setup-node2/**" + - ".github/workflows/sync-grammar-mirror.yml" + workflow_dispatch: + +concurrency: + group: sync-grammar-mirror + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + sync: + name: Sync to BoundaryML/textMate-baml + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup Node.js for typescript2 + uses: ./.github/actions/setup-node2 + + - name: Build grammar artifacts + working-directory: typescript2 + run: pnpm --filter @b/pkg-grammar build + + - name: Committed artifacts must match src/ + run: git diff --exit-code -- typescript2/pkg-grammar typescript2/app-vscode-ext + + - name: Run grammar tests + working-directory: typescript2 + run: pnpm --filter @b/pkg-grammar test + + - name: Clone mirror + env: + DEPLOY_KEY: ${{ secrets.TEXTMATE_BAML_DEPLOY_KEY }} + run: | + set -euo pipefail + mkdir -p ~/.ssh + printf '%s\n' "$DEPLOY_KEY" > ~/.ssh/textmate_baml + chmod 600 ~/.ssh/textmate_baml + ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null + export GIT_SSH_COMMAND="ssh -i ~/.ssh/textmate_baml -o IdentitiesOnly=yes" + git clone --depth 50 git@github.com:BoundaryML/textMate-baml.git /tmp/mirror-repo + + - name: Assemble and diff against mirror + id: diff + run: | + set -euo pipefail + # Stamp with the mirror's current version first, so the version field + # alone never registers as a content change (otherwise every rerun + # would bump and republish). + current="$(jq -r '.version // empty' /tmp/mirror-repo/package.json 2>/dev/null || true)" + node typescript2/pkg-grammar/scripts/assemble-mirror.mjs \ + --out /tmp/mirror-package \ + ${current:+--version "$current"} + rsync -a --delete --exclude .git /tmp/mirror-package/ /tmp/mirror-repo/ + if [ -n "$(git -C /tmp/mirror-repo status --porcelain)" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + git -C /tmp/mirror-repo status --porcelain + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Mirror already up to date." + fi + echo "current=${current}" >> "$GITHUB_OUTPUT" + + - name: Stamp next version and push + id: push + if: steps.diff.outputs.changed == 'true' + env: + CURRENT: ${{ steps.diff.outputs.current }} + SOURCE_SHA: ${{ github.sha }} + run: | + set -euo pipefail + # Next version: patch bump on whichever is ahead of npm latest and + # the mirror's own package.json (they can diverge if a publish + # failed); 0.1.0 if the package has never existed anywhere. + latest="$(npm view @boundaryml/baml-grammar version 2>/dev/null || true)" + base="$(printf '%s\n%s\n' "$latest" "$CURRENT" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)" + if [ -z "$base" ]; then + next="0.1.0" + else + next="$(echo "$base" | awk -F. '{printf "%d.%d.%d", $1, $2, $3 + 1}')" + fi + echo "version: base=${base:-} next=${next}" + jq --arg v "$next" '.version = $v' /tmp/mirror-repo/package.json > /tmp/pkg.json + mv /tmp/pkg.json /tmp/mirror-repo/package.json + + export GIT_SSH_COMMAND="ssh -i ~/.ssh/textmate_baml -o IdentitiesOnly=yes" + cd /tmp/mirror-repo + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -am "Sync from BoundaryML/baml@${SOURCE_SHA} (v${next})" + git push origin main + echo "version=${next}" >> "$GITHUB_OUTPUT" + + - name: Verify npm publish landed + if: steps.diff.outputs.changed == 'true' + env: + VERSION: ${{ steps.push.outputs.version }} + run: | + set -euo pipefail + echo "Waiting for @boundaryml/baml-grammar@${VERSION} to appear on npm (published by the mirror's publish.yml)..." + for i in $(seq 1 40); do + existing="$(npm view "@boundaryml/baml-grammar@${VERSION}" version 2>/dev/null || true)" + if [ "$existing" = "$VERSION" ]; then + echo "Published: @boundaryml/baml-grammar@${VERSION}" + exit 0 + fi + sleep 15 + done + echo "::error::@boundaryml/baml-grammar@${VERSION} did not appear on npm within 10 minutes." \ + "Check the publish workflow run in BoundaryML/textMate-baml." \ + "If this is the first publish: npm OIDC cannot create a new package —" \ + "run 'npm publish --access public' once from a mirror checkout, then configure the" \ + "trusted publisher (repo BoundaryML/textMate-baml, workflow publish.yml) in the npm package settings." + exit 1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 95b419a0ba9..108f0f957c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: entry: bash -c 'node typescript2/pkg-grammar/scripts/sync.mjs --check' language: system pass_filenames: false - files: ^typescript2/(pkg-grammar/.*\.tmLanguage\.json|app-vscode-ext/syntaxes/.*\.tmLanguage\.json)$ + files: ^typescript2/(pkg-grammar/(.*\.tmLanguage\.json|language-configuration\.json)|app-vscode-ext/(syntaxes/.*\.tmLanguage\.json|language-configuration\.json))$ priority: 0 - id: cargo-fmt diff --git a/typescript2/app-vscode-ext/language-configuration.json b/typescript2/app-vscode-ext/language-configuration.json index 6b56f4628d3..45904771d32 100644 --- a/typescript2/app-vscode-ext/language-configuration.json +++ b/typescript2/app-vscode-ext/language-configuration.json @@ -1,12 +1,9 @@ { "comments": { - // symbol used for single line comment. Remove this entry if your language does not support line comments "lineComment": "//", - // symbols used for start and end a block comment. Remove this entry if your language does not support block comments "blockComment": ["{/*", "*/}"] }, "brackets": [["{", "}"], ["[", "]"], ["(", ")"]], - // symbols that are auto closed when typing "autoClosingPairs": [ ["{", "}"], ["[", "]"], @@ -17,7 +14,6 @@ ["{#", "}"], ["{/*", "*/}"] ], - // symbols that can be used to surround a selection "surroundingPairs": [ ["{", "}"], ["[", "]"], diff --git a/typescript2/pkg-grammar/README.md b/typescript2/pkg-grammar/README.md index ea825fc86ab..a41c875c5ea 100644 --- a/typescript2/pkg-grammar/README.md +++ b/typescript2/pkg-grammar/README.md @@ -1,18 +1,25 @@ # @b/pkg-grammar -Canonical TextMate grammars for the BAML language — the **single source of -truth** for syntax highlighting. +Canonical TextMate grammar for the BAML language — the **single source of +truth** for syntax highlighting, everywhere: in this repo, on npm, and on +github.com. -The grammars are authored in TypeScript with +The grammar is authored in TypeScript with [`tmlanguage-generator`](https://www.npmjs.com/package/tmlanguage-generator) and compiled to JSON. You edit the typed sources, never the JSON. - `src/baml.ts` — the BAML grammar (`source.baml`), including prompt template highlighting. +- `language-configuration.json` — brackets/comments/auto-close pairs for + editor integrations (this one is hand-edited; it has no typed source). Generated artifacts (do not edit by hand): - `baml.tmLanguage.json` — emitted by the build, imported directly by consumers. +- `dist/index.js` + `dist/index.d.ts` — ESM module with the grammar inlined as + a JS object literal, typed as a Shiki `LanguageRegistration`. This is what + npm consumers import, so they never need JSON import attributes + (`with { type: "json" }`), which some bundlers (Metro) can't parse. ## Authoring @@ -25,16 +32,36 @@ Add repository rules in `src/*.ts` and reference them from `patterns` with pnpm --filter @b/pkg-grammar build ``` -`build` runs the generator (`scripts/build.ts`) and then `sync` to propagate the -JSON into the app-vscode-ext mirror. +`build` runs the generator (`scripts/build.ts`) and then `sync` to propagate +the JSON and `language-configuration.json` into the app-vscode-ext mirror. ## Consumers - **app-promptfiddle** / **pkg-editor** import the JSON directly (`import('@b/pkg-grammar/baml.tmLanguage.json')`) — no local copy. -- **app-vscode-ext** keeps a committed mirror under `syntaxes/` because VS Code - loads the grammar from the physical file named in `contributes.grammars[].path` - and the extension ships bundled (no `node_modules` in the `.vsix`). The mirror - is regenerated by `build` (run automatically on its `dev`/`build` via - `sync:grammar`) and a pre-commit hook (`pnpm --filter @b/pkg-grammar check`) - fails if it drifts. +- **app-vscode-ext** keeps committed mirrors (`syntaxes/baml.tmLanguage.json`, + `language-configuration.json`) because VS Code loads them from the physical + files named in `contributes` and the extension ships bundled (no + `node_modules` in the `.vsix`). The mirrors are regenerated by `build` and a + pre-commit hook (`pnpm --filter @b/pkg-grammar check`) fails if they drift. +- **[BoundaryML/textMate-baml](https://github.com/BoundaryML/textMate-baml)** + is an external mirror assembled by `scripts/assemble-mirror.mjs` from the + templates in `mirror/` and pushed by the `sync-grammar-mirror` workflow on + every grammar change on `canary`. It is never edited by hand. It serves + three consumers at once: + - **npm**: the mirror's own publish workflow releases it as + [`@boundaryml/baml-grammar`](https://www.npmjs.com/package/@boundaryml/baml-grammar). + - **GitHub Linguist** vendors it as a submodule for `.baml` highlighting on + github.com (picked up automatically at each Linguist release, roughly + quarterly). + - **Shiki's grammar registry** fetches the raw grammar from it weekly. + +## Don't break github.com + +Linguist compiles every vendored grammar with its own compiler, which converts +each Oniguruma regex to PCRE — regexes that work in Shiki/VS Code can fail +there. The `linguist-compile-gate` CI job runs that exact compiler against the +freshly built grammar, so a grammar change that would break github.com fails +the PR instead of surfacing months later at a Linguist release. Two invariants +Linguist pins that must never change: `scopeName: source.baml` and the `.baml` +file type (guarded in `tests/package-artifacts.test.ts`). diff --git a/typescript2/pkg-grammar/language-configuration.json b/typescript2/pkg-grammar/language-configuration.json new file mode 100644 index 00000000000..45904771d32 --- /dev/null +++ b/typescript2/pkg-grammar/language-configuration.json @@ -0,0 +1,34 @@ +{ + "comments": { + "lineComment": "//", + "blockComment": ["{/*", "*/}"] + }, + "brackets": [["{", "}"], ["[", "]"], ["(", ")"]], + "autoClosingPairs": [ + ["{", "}"], + ["[", "]"], + ["(", ")"], + { "open": "\"", "close": "\"" }, + ["#\"", "\"#"], + ["'", "'"], + ["{#", "}"], + ["{/*", "*/}"] + ], + "surroundingPairs": [ + ["{", "}"], + ["[", "]"], + ["(", ")"], + ["\"", "\""], + ["'", "'"] + ], + "onEnterRules": [ + { + "beforeText": "^\\s*(?:impl|class|function|enum|prompt).*?:\\s*$", + "action": { "indent": "indentOutdent" } + }, + { + "beforeText": "#\"", + "action": { "indent": "indentOutdent" } + } + ] +} diff --git a/typescript2/pkg-grammar/mirror/README.md b/typescript2/pkg-grammar/mirror/README.md new file mode 100644 index 00000000000..0bd63785015 --- /dev/null +++ b/typescript2/pkg-grammar/mirror/README.md @@ -0,0 +1,56 @@ +# BAML TextMate grammar + +The TextMate grammar for [BAML](https://github.com/BoundaryML/baml) (`scopeName: source.baml`), published as [`@boundaryml/baml-grammar`](https://www.npmjs.com/package/@boundaryml/baml-grammar). + +This repository is a read-only mirror, generated from [`typescript2/pkg-grammar`](https://github.com/BoundaryML/baml/tree/canary/typescript2/pkg-grammar) in the BAML monorepo. Do not edit it by hand; changes land here automatically when the grammar changes upstream. It is also the source [GitHub Linguist](https://github.com/github-linguist/linguist) vendors for `.baml` highlighting on github.com. + +## Install + +```sh +npm install @boundaryml/baml-grammar +``` + +## Shiki + +The default export is a ready-to-use `LanguageRegistration`: + +```ts +import { createHighlighter } from "shiki"; +import baml from "@boundaryml/baml-grammar"; + +const highlighter = await createHighlighter({ + themes: ["github-dark"], + langs: [baml], +}); + +highlighter.codeToHtml(code, { lang: "baml", theme: "github-dark" }); +``` + +The grammar is self-contained (it embeds no other language scopes), so no additional grammars need to be loaded. + +## VS Code / vscode-textmate / Monaco (monaco-textmate) + +Use the raw grammar JSON, plus the editor configuration (brackets, comments, auto-closing pairs): + +```ts +import grammar from "@boundaryml/baml-grammar/baml.tmLanguage.json" with { type: "json" }; +import languageConfiguration from "@boundaryml/baml-grammar/language-configuration.json" with { type: "json" }; +``` + +If your bundler cannot handle JSON import attributes (Metro/React Native, some Vite configs), use the default export instead: it is the same grammar inlined in a plain JS module. + +```ts +import grammar from "@boundaryml/baml-grammar"; +``` + +## Files + +| Path | Contents | +| --- | --- | +| `dist/index.js` | ESM module, default-exports the grammar (Shiki `LanguageRegistration`) | +| `grammars/baml.tmLanguage.json` | Raw TextMate grammar | +| `language-configuration.json` | VS Code style language configuration | + +## License + +Apache-2.0 diff --git a/typescript2/pkg-grammar/mirror/package.json b/typescript2/pkg-grammar/mirror/package.json new file mode 100644 index 00000000000..fe702ab25dc --- /dev/null +++ b/typescript2/pkg-grammar/mirror/package.json @@ -0,0 +1,40 @@ +{ + "name": "@boundaryml/baml-grammar", + "version": "0.0.0-dev", + "type": "module", + "description": "TextMate grammar for the BAML language, with a ready-to-use Shiki LanguageRegistration export.", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/BoundaryML/textMate-baml.git" + }, + "homepage": "https://www.boundaryml.com", + "bugs": { + "url": "https://github.com/BoundaryML/baml/issues" + }, + "keywords": [ + "baml", + "textmate", + "tmlanguage", + "shiki", + "syntax-highlighting", + "grammar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./baml.tmLanguage.json": "./grammars/baml.tmLanguage.json", + "./language-configuration.json": "./language-configuration.json", + "./package.json": "./package.json" + }, + "files": [ + "dist", + "grammars", + "language-configuration.json" + ], + "dependencies": { + "@shikijs/types": "^3.0.0" + } +} diff --git a/typescript2/pkg-grammar/mirror/publish.yml b/typescript2/pkg-grammar/mirror/publish.yml new file mode 100644 index 00000000000..ae2cdd1ee7b --- /dev/null +++ b/typescript2/pkg-grammar/mirror/publish.yml @@ -0,0 +1,81 @@ +# Publishes @boundaryml/baml-grammar to npm on every push to main. +# +# This file is generated from typescript2/pkg-grammar/mirror/publish.yml in +# BoundaryML/baml and synced here by that repo's sync-grammar-mirror workflow. +# Do not edit it in this repository. +# +# Auth is npm OIDC trusted publishing: the npm package settings must list this +# repo (BoundaryML/textMate-baml) and this workflow filename (publish.yml) as a +# trusted publisher. No token secrets are involved. The first-ever publish of +# the package cannot use OIDC and must be done once by hand (npm publish from a +# checkout of this repo); after that, configure the trusted publisher and this +# workflow takes over. + +name: Publish to npm + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/setup-node@v4 + with: + node-version: latest + registry-url: https://registry.npmjs.org + + - name: Ensure npm CLI supports trusted publishing + run: | + set -euo pipefail + required_version="11.5.1" + current_version="$(npm --version)" + if [ "$(printf '%s\n' "$required_version" "$current_version" | sort -V | head -n1)" != "$required_version" ]; then + echo "npm $current_version is lower than required $required_version (needed for OIDC trusted publishing)" >&2 + exit 1 + fi + + - name: Smoke-test the artifact + run: | + set -euo pipefail + node --input-type=module -e ' + import assert from "node:assert/strict"; + import { readFileSync } from "node:fs"; + const grammar = (await import("./dist/index.js")).default; + assert.equal(grammar.scopeName, "source.baml"); + assert.equal(grammar.name, "baml"); + const raw = JSON.parse(readFileSync("grammars/baml.tmLanguage.json", "utf8")); + assert.deepEqual(grammar, raw, "dist/index.js drifted from grammars/baml.tmLanguage.json"); + const config = JSON.parse(readFileSync("language-configuration.json", "utf8")); + assert.ok(config.brackets.length > 0); + const pkg = JSON.parse(readFileSync("package.json", "utf8")); + assert.equal(pkg.name, "@boundaryml/baml-grammar"); + assert.notEqual(pkg.version, "0.0.0-dev", "package.json version was never stamped"); + console.log("smoke ok:", pkg.name, pkg.version); + ' + + - name: Publish + run: | + set -euo pipefail + name="$(jq -r '.name' package.json)" + version="$(jq -r '.version' package.json)" + # Reruns and no-op pushes must not fail on "version already exists": + # skip when this exact version is already on the registry. (`npm view` + # exits non-zero when the package has never been published; treat + # that as "not published yet".) + existing="$(npm view "${name}@${version}" version 2>/dev/null || true)" + if [ "$existing" = "$version" ]; then + echo "${name}@${version} is already published; nothing to do." + exit 0 + fi + npm publish --access public diff --git a/typescript2/pkg-grammar/package.json b/typescript2/pkg-grammar/package.json index 8991cdb52b1..6acdd89677b 100644 --- a/typescript2/pkg-grammar/package.json +++ b/typescript2/pkg-grammar/package.json @@ -3,22 +3,31 @@ "version": "0.1.0", "private": true, "type": "module", - "description": "Canonical TextMate grammars for the BAML language (single source of truth, consumed by app-vscode-ext and app-promptfiddle).", + "description": "Canonical TextMate grammar for the BAML language (single source of truth, consumed by app-vscode-ext and app-promptfiddle, and published to npm as @boundaryml/baml-grammar via the boundaryml/textMate-baml mirror).", "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, "./baml.tmLanguage.json": "./baml.tmLanguage.json", + "./language-configuration.json": "./language-configuration.json", "./package.json": "./package.json" }, "files": [ - "*.tmLanguage.json" + "*.tmLanguage.json", + "language-configuration.json", + "dist" ], "scripts": { "build": "tsx scripts/build.ts && node scripts/sync.mjs", "sync": "node scripts/sync.mjs", "check": "node scripts/sync.mjs --check", + "assemble": "node scripts/assemble-mirror.mjs", "preview": "vite", "test": "vitest run --config vitest.config.ts" }, "devDependencies": { + "@shikijs/types": "^3.0.0", "shiki": "^3.0.0", "tmlanguage-generator": "^0.6.8", "tsx": "^4.19.2", diff --git a/typescript2/pkg-grammar/scripts/assemble-mirror.mjs b/typescript2/pkg-grammar/scripts/assemble-mirror.mjs new file mode 100644 index 00000000000..6f04469c338 --- /dev/null +++ b/typescript2/pkg-grammar/scripts/assemble-mirror.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Assemble the full contents of the BoundaryML/textMate-baml mirror repo, +// which is both the npm package @boundaryml/baml-grammar (published by the +// mirror's own publish workflow) and the grammar source GitHub Linguist +// vendors for .baml highlighting on github.com. +// +// The output directory is the complete desired state of the mirror: the +// sync-grammar-mirror workflow rsyncs it over a mirror checkout with --delete +// (excluding .git), so anything not emitted here gets removed from the mirror. +// +// Usage: +// node scripts/assemble-mirror.mjs --out [--version ] +// +// Requires a prior `pnpm --filter @b/pkg-grammar build` (dist/ must exist). +// Without --version the template's 0.0.0-dev placeholder is kept; the sync +// workflow always stamps a real version. + +import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +const here = dirname(fileURLToPath(import.meta.url)); +const pkgRoot = resolve(here, '..'); +const repoRoot = resolve(pkgRoot, '../..'); + +const { values } = parseArgs({ + options: { + out: { type: 'string' }, + version: { type: 'string' }, + }, +}); + +if (!values.out) { + console.error('usage: assemble-mirror.mjs --out [--version ]'); + process.exit(1); +} +if (values.version && !/^\d+\.\d+\.\d+$/.test(values.version)) { + console.error(`invalid --version: ${values.version}`); + process.exit(1); +} + +const distDir = resolve(pkgRoot, 'dist'); +if (!existsSync(resolve(distDir, 'index.js'))) { + console.error('dist/index.js missing; run `pnpm --filter @b/pkg-grammar build` first'); + process.exit(1); +} + +const out = resolve(process.cwd(), values.out); +rmSync(out, { recursive: true, force: true }); +mkdirSync(resolve(out, 'grammars'), { recursive: true }); +mkdirSync(resolve(out, '.github/workflows'), { recursive: true }); + +// Package artifacts. +cpSync(distDir, resolve(out, 'dist'), { recursive: true }); +cpSync(resolve(pkgRoot, 'baml.tmLanguage.json'), resolve(out, 'grammars/baml.tmLanguage.json')); +cpSync(resolve(pkgRoot, 'language-configuration.json'), resolve(out, 'language-configuration.json')); + +// Repo scaffolding. Linguist's license check needs LICENSE; publish.yml is the +// mirror's own npm release workflow. +cpSync(resolve(repoRoot, 'LICENSE'), resolve(out, 'LICENSE')); +cpSync(resolve(pkgRoot, 'mirror/README.md'), resolve(out, 'README.md')); +cpSync(resolve(pkgRoot, 'mirror/publish.yml'), resolve(out, '.github/workflows/publish.yml')); + +// package.json, with the version stamped in. +const pkg = JSON.parse(readFileSync(resolve(pkgRoot, 'mirror/package.json'), 'utf8')); +if (values.version) { + pkg.version = values.version; +} +writeFileSync(resolve(out, 'package.json'), `${JSON.stringify(pkg, null, 2)}\n`); + +console.log(`assembled mirror at ${out} (version ${pkg.version})`); diff --git a/typescript2/pkg-grammar/scripts/build.ts b/typescript2/pkg-grammar/scripts/build.ts index e775031eb47..09eea3eeb96 100644 --- a/typescript2/pkg-grammar/scripts/build.ts +++ b/typescript2/pkg-grammar/scripts/build.ts @@ -3,8 +3,15 @@ // The emitted JSON files are the artifacts every consumer reads // (app-promptfiddle, pkg-editor) and the source for the app-vscode-ext mirror // (see scripts/sync.mjs). Run via `pnpm --filter @b/pkg-grammar build`. +// +// Also emits dist/index.js + dist/index.d.ts with the grammar inlined as a JS +// object literal. Consumers of the published package import that instead of +// the raw JSON: JSON imports need `with { type: "json" }` under +// moduleResolution nodenext, and some bundlers (Metro/React Native) don't +// parse that attribute, so the JS module keeps JSON out of the consumer +// boundary entirely. -import { writeFileSync } from "node:fs"; +import { mkdirSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { emitJSON, type Grammar } from "tmlanguage-generator"; @@ -21,4 +28,44 @@ for (const { grammar, out } of GRAMMARS) { const json = await emitJSON(grammar, { errorSourceFilePath: out }); writeFileSync(resolve(pkgRoot, out), json); console.log(`generated ${out}`); + + if (out !== "baml.tmLanguage.json") { + continue; + } + + // Round-trip through JSON.parse/stringify so the inlined literal is exactly + // the parsed value of the raw JSON artifact (the drift test relies on this). + const literal = JSON.stringify(JSON.parse(json), null, 2); + const distDir = resolve(pkgRoot, "dist"); + mkdirSync(distDir, { recursive: true }); + + writeFileSync( + resolve(distDir, "index.js"), + `// Generated by scripts/build.ts from src/baml.ts. Do not edit. +// The grammar is inlined as a JS object literal so consumers never import +// JSON (avoids \`with { type: "json" }\` import-attribute incompatibilities +// across Vite, Metro, and nodenext). +const grammar = ${literal}; + +export const bamlLanguage = grammar; +export default bamlLanguage; +`, + ); + + writeFileSync( + resolve(distDir, "index.d.ts"), + `// Generated by scripts/build.ts from src/baml.ts. Do not edit. +import type { LanguageRegistration } from "@shikijs/types"; + +/** + * The BAML TextMate grammar (\`scopeName: "source.baml"\`) as a Shiki + * \`LanguageRegistration\`, also usable anywhere a raw TextMate grammar + * object is accepted (vscode-textmate, monaco-textmate, ...). + */ +export declare const bamlLanguage: LanguageRegistration; +export default bamlLanguage; +`, + ); + + console.log("generated dist/index.js, dist/index.d.ts"); } diff --git a/typescript2/pkg-grammar/scripts/sync.mjs b/typescript2/pkg-grammar/scripts/sync.mjs index 045cb83c1c7..0f5def4a15e 100644 --- a/typescript2/pkg-grammar/scripts/sync.mjs +++ b/typescript2/pkg-grammar/scripts/sync.mjs @@ -6,10 +6,11 @@ // named in `contributes.grammars[].path`, and app-vscode-ext is bundled with // tsup (no node_modules shipped in the .vsix), so the extension can't point at // @b/pkg-grammar at runtime — it needs the JSON committed under syntaxes/. +// Same story for language-configuration.json (`contributes.languages[].configuration`). // app-promptfiddle has no such mirror; it imports the package directly. // // Usage: -// node scripts/sync.mjs Copy canonical grammars into the mirror. +// node scripts/sync.mjs Copy canonical files into the mirror. // node scripts/sync.mjs --check Exit non-zero if the mirror has drifted // (used by the pre-commit hook / CI). @@ -19,16 +20,20 @@ import { fileURLToPath } from 'node:url'; const here = dirname(fileURLToPath(import.meta.url)); const pkgRoot = resolve(here, '..'); -const mirrorDir = resolve(pkgRoot, '../app-vscode-ext/syntaxes'); +const extRoot = resolve(pkgRoot, '../app-vscode-ext'); -const GRAMMARS = ['baml.tmLanguage.json']; +// canonical path (relative to pkg-grammar) -> mirror path (relative to app-vscode-ext) +const MIRRORED = [ + ['baml.tmLanguage.json', 'syntaxes/baml.tmLanguage.json'], + ['language-configuration.json', 'language-configuration.json'], +]; const check = process.argv.includes('--check'); let drifted = false; -for (const name of GRAMMARS) { +for (const [name, mirrorRel] of MIRRORED) { const canonical = readFileSync(resolve(pkgRoot, name), 'utf8'); - const mirrorPath = resolve(mirrorDir, name); + const mirrorPath = resolve(extRoot, mirrorRel); if (check) { let mirror = null; @@ -39,15 +44,15 @@ for (const name of GRAMMARS) { } if (mirror !== canonical) { drifted = true; - console.error(`✗ ${name}: app-vscode-ext/syntaxes is out of sync with @b/pkg-grammar`); + console.error(`✗ ${name}: app-vscode-ext/${mirrorRel} is out of sync with @b/pkg-grammar`); } } else { writeFileSync(mirrorPath, canonical); - console.log(`✓ synced ${name} → app-vscode-ext/syntaxes`); + console.log(`✓ synced ${name} → app-vscode-ext/${mirrorRel}`); } } if (check && drifted) { - console.error('\nGrammars live in typescript2/pkg-grammar. Run:\n pnpm --filter @b/pkg-grammar sync\nthen commit the regenerated app-vscode-ext/syntaxes/*.tmLanguage.json.'); + console.error('\nThese files live in typescript2/pkg-grammar. Run:\n pnpm --filter @b/pkg-grammar sync\nthen commit the regenerated copies in app-vscode-ext.'); process.exit(1); } diff --git a/typescript2/pkg-grammar/tests/package-artifacts.test.ts b/typescript2/pkg-grammar/tests/package-artifacts.test.ts new file mode 100644 index 00000000000..c0317d9c400 --- /dev/null +++ b/typescript2/pkg-grammar/tests/package-artifacts.test.ts @@ -0,0 +1,105 @@ +// Guards on the artifacts the published package (and GitHub, via the +// boundaryml/textMate-baml mirror) consumes: grammar invariants that external +// registries pin, the dist/ ESM module, and language-configuration.json. + +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeAll, describe, expect, it } from "vitest"; +import { createHighlighter } from "shiki"; +import rawGrammar from "../baml.tmLanguage.json"; + +const pkgRoot = join(import.meta.dirname, ".."); +const distPath = join(pkgRoot, "dist", "index.js"); + +describe("grammar invariants pinned by external consumers", () => { + // GitHub Linguist's languages.yml maps BAML to tm_scope source.baml and + // extension .baml. Changing either doesn't fail anywhere loudly — .baml + // files on github.com just silently render as plain text. + it("keeps scopeName source.baml", () => { + expect(rawGrammar.scopeName).toBe("source.baml"); + }); + + it("keeps name baml", () => { + expect(rawGrammar.name).toBe("baml"); + }); + + it("keeps fileTypes [baml]", () => { + expect(rawGrammar.fileTypes).toContain("baml"); + }); + + it("stays self-contained (no external grammar scopes embedded)", () => { + const json = readFileSync(join(pkgRoot, "baml.tmLanguage.json"), "utf8"); + const includes = [...json.matchAll(/"include":\s*"([^"#$][^"]*)"/g)].map( + (m) => m[1], + ); + // Includes must be internal (#repo / $self / $base); an external scope + // would require consumers to load a second grammar, which Shiki and + // Linguist won't do for us. + expect(includes).toEqual([]); + }); +}); + +describe("dist ESM artifact", () => { + let distDefault: unknown; + + beforeAll(async () => { + expect( + existsSync(distPath), + "dist/index.js missing — run `pnpm --filter @b/pkg-grammar build` first", + ).toBe(true); + distDefault = (await import(distPath)).default; + }); + + it("default export deep-equals the raw tmLanguage JSON", () => { + // The mirror publishes both; if the inlined literal ever drifts from the + // raw JSON, npm consumers and GitHub would highlight differently. + expect(distDefault).toEqual(JSON.parse(JSON.stringify(rawGrammar))); + }); + + it("works as a Shiki LanguageRegistration end-to-end", async () => { + const highlighter = await createHighlighter({ + themes: ["github-dark"], + langs: [distDefault as never], + }); + + const sample = [ + "// greet the user", + "function Greet(name: string) -> string {", + " client GPT4", + ' prompt #"Hello {{ name }}"#', + "}", + "", + "enum Mood { Happy Sad }", + ].join("\n"); + + const { tokens } = highlighter.codeToTokens(sample, { + lang: "baml", + theme: "github-dark", + includeExplanation: "scopeName", + }); + + const scopes = tokens + .flat() + .flatMap((token) => token.explanation ?? []) + .flatMap((part) => part.scopes ?? []) + .map((scope) => scope.scopeName); + + expect(scopes).toContain("source.baml"); + // At least something beyond the root scope must match, otherwise the + // grammar is loaded but effectively inert. + expect(scopes.some((s) => s !== "source.baml")).toBe(true); + highlighter.dispose(); + }); +}); + +describe("language-configuration.json", () => { + it("is strict JSON (npm consumers JSON.parse it; no JSONC comments)", () => { + const raw = readFileSync( + join(pkgRoot, "language-configuration.json"), + "utf8", + ); + const config = JSON.parse(raw); + expect(config.comments.lineComment).toBe("//"); + expect(Array.isArray(config.brackets)).toBe(true); + }); +}); diff --git a/typescript2/pkg-grammar/tests/repo-corpus.test.ts b/typescript2/pkg-grammar/tests/repo-corpus.test.ts new file mode 100644 index 00000000000..4f0c26fb268 --- /dev/null +++ b/typescript2/pkg-grammar/tests/repo-corpus.test.ts @@ -0,0 +1,63 @@ +// Tokenize every .baml file in the repository, not just the curated fixtures. +// The snapshot suite catches scope regressions; this catches the failure modes +// that matter for GitHub and other bulk consumers: tokenizer crashes and +// catastrophic-backtracking regexes that make highlighting hang or bail on +// real-world files. + +import { execSync } from "node:child_process"; +import { readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { createHighlighter } from "shiki"; +import bamlGrammar from "../baml.tmLanguage.json"; + +const THEME = "github-dark"; +// Skip the multi-MB LFS stress files; everything a human would read stays in. +const MAX_BYTES = 256 * 1024; +// Generous per-file wall-clock budget (slow CI runners included). A +// pathological regex blows way past this; normal files tokenize in +// milliseconds. +const PER_FILE_TIMEOUT_MS = 15_000; + +const repoRoot = join(import.meta.dirname, "..", "..", ".."); + +const files = execSync("git ls-files -z -- '*.baml'", { + cwd: repoRoot, + maxBuffer: 64 * 1024 * 1024, +}) + .toString("utf8") + .split("\0") + .filter(Boolean) + .filter((path) => { + try { + return statSync(join(repoRoot, path)).size <= MAX_BYTES; + } catch { + return false; + } + }); + +const highlighter = await createHighlighter({ + themes: [THEME], + langs: [bamlGrammar as never], +}); + +describe(`repo corpus (${files.length} .baml files)`, () => { + it("found a non-trivial corpus", () => { + expect(files.length).toBeGreaterThan(100); + }); + + for (const file of files) { + it( + `tokenizes ${file}`, + () => { + const source = readFileSync(join(repoRoot, file), "utf8"); + const { tokens } = highlighter.codeToTokens(source, { + lang: "baml", + theme: THEME, + }); + expect(tokens.length).toBeGreaterThan(0); + }, + PER_FILE_TIMEOUT_MS, + ); + } +}); diff --git a/typescript2/pnpm-lock.yaml b/typescript2/pnpm-lock.yaml index f1fcc07f071..ef4a828a906 100644 --- a/typescript2/pnpm-lock.yaml +++ b/typescript2/pnpm-lock.yaml @@ -480,6 +480,9 @@ importers: pkg-grammar: devDependencies: + '@shikijs/types': + specifier: ^3.0.0 + version: 3.23.0 shiki: specifier: ^3.0.0 version: 3.23.0 From a523e757267f592738ee6deaef1faa3fa74b8d3a Mon Sep 17 00:00:00 2001 From: hellovai Date: Sun, 12 Jul 2026 00:37:36 -0700 Subject: [PATCH 2/2] fix(grammar): auto-close {# with #} in language configuration {# ... #} is the template comment form, so typing {# should insert #}, not a bare brace. Co-Authored-By: Claude Fable 5 --- typescript2/app-vscode-ext/language-configuration.json | 2 +- typescript2/pkg-grammar/language-configuration.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/typescript2/app-vscode-ext/language-configuration.json b/typescript2/app-vscode-ext/language-configuration.json index 45904771d32..1600e87c747 100644 --- a/typescript2/app-vscode-ext/language-configuration.json +++ b/typescript2/app-vscode-ext/language-configuration.json @@ -11,7 +11,7 @@ { "open": "\"", "close": "\"" }, ["#\"", "\"#"], ["'", "'"], - ["{#", "}"], + ["{#", "#}"], ["{/*", "*/}"] ], "surroundingPairs": [ diff --git a/typescript2/pkg-grammar/language-configuration.json b/typescript2/pkg-grammar/language-configuration.json index 45904771d32..1600e87c747 100644 --- a/typescript2/pkg-grammar/language-configuration.json +++ b/typescript2/pkg-grammar/language-configuration.json @@ -11,7 +11,7 @@ { "open": "\"", "close": "\"" }, ["#\"", "\"#"], ["'", "'"], - ["{#", "}"], + ["{#", "#}"], ["{/*", "*/}"] ], "surroundingPairs": [