hellovai · Jul 12, 2026 · Jul 12, 2026 · Jul 12, 2026
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -219,6 +219,7 @@ jobs:
           if git diff --quiet "${MERGE_BASE}...HEAD" -- \
             ':typescript2/pkg-grammar/**' \
             ':typescript2/app-vscode-ext/syntaxes/**' \
+            ':typescript2/app-vscode-ext/language-configuration.json' \
             ':typescript2/app-vscode-ext/package.json' \
             ':typescript2/pnpm-lock.yaml' \
             ':.github/actions/setup-node2/**' \
@@ -502,10 +503,79 @@ jobs:
       - name: "Setup Node.js for typescript2"
         uses: ./.github/actions/setup-node2
 
-      - name: "Run grammar snapshot tests"
+      - name: "Build grammar artifacts"
+        run: pnpm --filter @b/pkg-grammar build
+        working-directory: typescript2
+
+      - name: "Committed artifacts must match src/"
+        # build regenerates baml.tmLanguage.json (+ the app-vscode-ext copies
+        # via sync.mjs); any diff means someone edited src/ without committing
+        # the regenerated artifacts, or edited an artifact by hand.
+        run: git diff --exit-code -- typescript2/pkg-grammar typescript2/app-vscode-ext
+
+      - name: "Run grammar tests"
         run: pnpm --filter @b/pkg-grammar test
         working-directory: typescript2
 
+  linguist-compile-gate:
+    name: "Linguist grammar-compiler gate"
+    runs-on: ubuntu-latest
+    needs: determine_changes
+    if: needs.determine_changes.outputs.grammar == 'true' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/canary'
+    steps:
+      - name: "Checkout Branch"
+        uses: useblacksmith/checkout@v1
+        with:
+          persist-credentials: false
+
+      - name: "Setup Node.js for typescript2"
+        uses: ./.github/actions/setup-node2
+
+      - name: "Build grammar artifacts"
+        run: pnpm --filter @b/pkg-grammar build
+        working-directory: typescript2
+
+      - name: "Checkout github-linguist/linguist"
+        env:
+          # GitHub's release process compiles every vendored grammar with this
+          # tool; running the same compile here means a grammar that passes CI
+          # cannot break .baml highlighting on github.com when Linguist next
+          # pulls the BoundaryML/textMate-baml mirror. Bump the ref alongside
+          # Linguist releases.
+          LINGUIST_REF: v9.6.0
+        run: |
+          git clone --depth 1 --branch "$LINGUIST_REF" \
+            https://github.com/github-linguist/linguist.git /tmp/linguist
+
+      - name: "Stage candidate grammar as the textMate-baml vendor dir"
+        run: |
+          set -euo pipefail
+          # Drop the registered entry for our vendor dir so the compiler treats
+          # the staged grammar as a fresh add (avoids duplicate-scope errors
+          # against the currently-vendored copy).
+          python3 - <<'EOF'
+          import re
+          path = "/tmp/linguist/grammars.yml"
+          with open(path) as f:
+              content = f.read()
+          content = re.sub(r"vendor/grammars/textMate-baml:\n(- [^\n]+\n)+", "", content)
+          with open(path, "w") as f:
+              f.write(content)
+          EOF
+          rm -rf /tmp/linguist/vendor/grammars/textMate-baml
+          mkdir -p /tmp/linguist/vendor/grammars/textMate-baml/grammars
+          cp typescript2/pkg-grammar/baml.tmLanguage.json \
+            /tmp/linguist/vendor/grammars/textMate-baml/grammars/
+
+      - name: "Compile with Linguist's grammar-compiler"
+        working-directory: /tmp/linguist
+        run: |
+          set -euo pipefail
+          export USER="${USER:-$(whoami)}"
+          # No -f: any regex that Linguist's Oniguruma->PCRE conversion cannot
+          # handle must fail this job, not surface at Linguist's next release.
+          script/grammar-compiler add vendor/grammars/textMate-baml
+
   miri-tests:
     name: "Miri (unsafe code verification)"
     runs-on: blacksmith-4vcpu-ubuntu-2404
@@ -958,6 +1028,7 @@ jobs:
       - docs
       - webview-tests
       - grammar-tests
+      - linguist-compile-gate
       # - miri-tests  # TEMPORARILY DISABLED: miri tests timing out
       - prof-concurrency
       - proto-sync
@@ -984,6 +1055,7 @@ jobs:
           echo "| docs | ${{ needs.docs.result }} |" >> $GITHUB_STEP_SUMMARY
           echo "| webview-tests | ${{ needs.webview-tests.result }} |" >> $GITHUB_STEP_SUMMARY
           echo "| grammar-tests | ${{ needs.grammar-tests.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| linguist-compile-gate | ${{ needs.linguist-compile-gate.result }} |" >> $GITHUB_STEP_SUMMARY
           # echo "| miri-tests | ${{ needs.miri-tests.result }} |" >> $GITHUB_STEP_SUMMARY  # TEMPORARILY DISABLED
           echo "| prof-concurrency | ${{ needs.prof-concurrency.result }} |" >> $GITHUB_STEP_SUMMARY
           echo "| proto-sync | ${{ needs.proto-sync.result }} |" >> $GITHUB_STEP_SUMMARY

diff --git a/.github/workflows/sync-grammar-mirror.yml b/.github/workflows/sync-grammar-mirror.yml
@@ -0,0 +1,146 @@
+# Sync the BAML TextMate grammar to the BoundaryML/textMate-baml mirror repo.
+#
+# The mirror is the single external artifact for every third-party consumer of
+# the grammar: it is the npm package @boundaryml/baml-grammar (published by the
+# mirror's own publish.yml, which this sync also ships), the repo GitHub
+# Linguist vendors for .baml highlighting on github.com, and the source URL the
+# Shiki grammar registry fetches from. Grammar edits land in
+# typescript2/pkg-grammar; this workflow propagates them, so the mirror must
+# never be edited by hand.
+#
+# Auth: the TEXTMATE_BAML_DEPLOY_KEY secret holds an SSH private key whose
+# public half is a write-access deploy key on BoundaryML/textMate-baml.
+# (A deploy key, not a PAT, so pushes may update the mirror's workflow files.)
+#
+# Bootstrap note: npm OIDC trusted publishing cannot create a brand-new
+# package. The first-ever npm publish must be done once by hand from a mirror
+# checkout, then the trusted publisher (repo BoundaryML/textMate-baml, workflow
+# publish.yml) configured in the npm package settings. Until then the final
+# verify step here fails; that is expected and is the reminder to finish setup.
+
+name: Sync grammar mirror
+
+on:
+  push:
+    branches: [canary]
+    paths:
+      - "typescript2/pkg-grammar/**"
+      - ".github/actions/setup-node2/**"
+      - ".github/workflows/sync-grammar-mirror.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: sync-grammar-mirror
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  sync:
+    name: Sync to BoundaryML/textMate-baml
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Setup Node.js for typescript2
+        uses: ./.github/actions/setup-node2
+
+      - name: Build grammar artifacts
+        working-directory: typescript2
+        run: pnpm --filter @b/pkg-grammar build
+
+      - name: Committed artifacts must match src/
+        run: git diff --exit-code -- typescript2/pkg-grammar typescript2/app-vscode-ext
+
+      - name: Run grammar tests
+        working-directory: typescript2
+        run: pnpm --filter @b/pkg-grammar test
+
+      - name: Clone mirror
+        env:
+          DEPLOY_KEY: ${{ secrets.TEXTMATE_BAML_DEPLOY_KEY }}
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.ssh
+          printf '%s\n' "$DEPLOY_KEY" > ~/.ssh/textmate_baml
+          chmod 600 ~/.ssh/textmate_baml
+          ssh-keyscan github.com >> ~/.ssh/known_hosts 2>/dev/null
+          export GIT_SSH_COMMAND="ssh -i ~/.ssh/textmate_baml -o IdentitiesOnly=yes"
+          git clone --depth 50 [email protected]:BoundaryML/textMate-baml.git /tmp/mirror-repo
+
+      - name: Assemble and diff against mirror
+        id: diff
+        run: |
+          set -euo pipefail
+          # Stamp with the mirror's current version first, so the version field
+          # alone never registers as a content change (otherwise every rerun
+          # would bump and republish).
+          current="$(jq -r '.version // empty' /tmp/mirror-repo/package.json 2>/dev/null || true)"
+          node typescript2/pkg-grammar/scripts/assemble-mirror.mjs \
+            --out /tmp/mirror-package \
+            ${current:+--version "$current"}
+          rsync -a --delete --exclude .git /tmp/mirror-package/ /tmp/mirror-repo/
+          if [ -n "$(git -C /tmp/mirror-repo status --porcelain)" ]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            git -C /tmp/mirror-repo status --porcelain
+          else
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "Mirror already up to date."
+          fi
+          echo "current=${current}" >> "$GITHUB_OUTPUT"
+
+      - name: Stamp next version and push
+        id: push
+        if: steps.diff.outputs.changed == 'true'
+        env:
+          CURRENT: ${{ steps.diff.outputs.current }}
+          SOURCE_SHA: ${{ github.sha }}
+        run: |
+          set -euo pipefail
+          # Next version: patch bump on whichever is ahead of npm latest and
+          # the mirror's own package.json (they can diverge if a publish
+          # failed); 0.1.0 if the package has never existed anywhere.
+          latest="$(npm view @boundaryml/baml-grammar version 2>/dev/null || true)"
+          base="$(printf '%s\n%s\n' "$latest" "$CURRENT" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)"
+          if [ -z "$base" ]; then
+            next="0.1.0"
+          else
+            next="$(echo "$base" | awk -F. '{printf "%d.%d.%d", $1, $2, $3 + 1}')"
+          fi
+          echo "version: base=${base:-<none>} next=${next}"
+          jq --arg v "$next" '.version = $v' /tmp/mirror-repo/package.json > /tmp/pkg.json
+          mv /tmp/pkg.json /tmp/mirror-repo/package.json
+
+          export GIT_SSH_COMMAND="ssh -i ~/.ssh/textmate_baml -o IdentitiesOnly=yes"
+          cd /tmp/mirror-repo
+          git -c user.name="github-actions[bot]" \
+              -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
+              commit -am "Sync from BoundaryML/baml@${SOURCE_SHA} (v${next})"
+          git push origin main
+          echo "version=${next}" >> "$GITHUB_OUTPUT"
+
+      - name: Verify npm publish landed
+        if: steps.diff.outputs.changed == 'true'
+        env:
+          VERSION: ${{ steps.push.outputs.version }}
+        run: |
+          set -euo pipefail
+          echo "Waiting for @boundaryml/baml-grammar@${VERSION} to appear on npm (published by the mirror's publish.yml)..."
+          for i in $(seq 1 40); do
+            existing="$(npm view "@boundaryml/baml-grammar@${VERSION}" version 2>/dev/null || true)"
+            if [ "$existing" = "$VERSION" ]; then
+              echo "Published: @boundaryml/baml-grammar@${VERSION}"
+              exit 0
+            fi
+            sleep 15
+          done
+          echo "::error::@boundaryml/baml-grammar@${VERSION} did not appear on npm within 10 minutes." \
+            "Check the publish workflow run in BoundaryML/textMate-baml." \
+            "If this is the first publish: npm OIDC cannot create a new package —" \
+            "run 'npm publish --access public' once from a mirror checkout, then configure the" \
+            "trusted publisher (repo BoundaryML/textMate-baml, workflow publish.yml) in the npm package settings."
+          exit 1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -46,7 +46,7 @@ repos:
         entry: bash -c 'node typescript2/pkg-grammar/scripts/sync.mjs --check'
         language: system
         pass_filenames: false
-        files: ^typescript2/(pkg-grammar/.*\.tmLanguage\.json|app-vscode-ext/syntaxes/.*\.tmLanguage\.json)$
+        files: ^typescript2/(pkg-grammar/(.*\.tmLanguage\.json|language-configuration\.json)|app-vscode-ext/(syntaxes/.*\.tmLanguage\.json|language-configuration\.json))$
         priority: 0
 
       - id: cargo-fmt

diff --git a/typescript2/app-vscode-ext/language-configuration.json b/typescript2/app-vscode-ext/language-configuration.json
@@ -1,23 +1,19 @@
 {
   "comments": {
-    // symbol used for single line comment. Remove this entry if your language does not support line comments
     "lineComment": "//",
-    // symbols used for start and end a block comment. Remove this entry if your language does not support block comments
     "blockComment": ["{/*", "*/}"]
   },
   "brackets": [["{", "}"], ["[", "]"], ["(", ")"]],
-  // symbols that are auto closed when typing
   "autoClosingPairs": [
     ["{", "}"],
     ["[", "]"],
     ["(", ")"],
     { "open": "\"", "close": "\"" },
     ["#\"", "\"#"],
     ["'", "'"],
-    ["{#", "}"],
+    ["{#", "#}"],
     ["{/*", "*/}"]
   ],
-  // symbols that can be used to surround a selection
   "surroundingPairs": [
     ["{", "}"],
     ["[", "]"],

diff --git a/typescript2/pkg-grammar/README.md b/typescript2/pkg-grammar/README.md
@@ -1,18 +1,25 @@
 # @b/pkg-grammar
 
-Canonical TextMate grammars for the BAML language — the **single source of
-truth** for syntax highlighting.
+Canonical TextMate grammar for the BAML language — the **single source of
+truth** for syntax highlighting, everywhere: in this repo, on npm, and on
+github.com.
 
-The grammars are authored in TypeScript with
+The grammar is authored in TypeScript with
 [`tmlanguage-generator`](https://www.npmjs.com/package/tmlanguage-generator)
 and compiled to JSON. You edit the typed sources, never the JSON.
 
 - `src/baml.ts` — the BAML grammar (`source.baml`), including prompt template
   highlighting.
+- `language-configuration.json` — brackets/comments/auto-close pairs for
+  editor integrations (this one is hand-edited; it has no typed source).
 
 Generated artifacts (do not edit by hand):
 
 - `baml.tmLanguage.json` — emitted by the build, imported directly by consumers.
+- `dist/index.js` + `dist/index.d.ts` — ESM module with the grammar inlined as
+  a JS object literal, typed as a Shiki `LanguageRegistration`. This is what
+  npm consumers import, so they never need JSON import attributes
+  (`with { type: "json" }`), which some bundlers (Metro) can't parse.
 
 ## Authoring
 
@@ -25,16 +32,36 @@ Add repository rules in `src/*.ts` and reference them from `patterns` with
 pnpm --filter @b/pkg-grammar build
 ```
 
-`build` runs the generator (`scripts/build.ts`) and then `sync` to propagate the
-JSON into the app-vscode-ext mirror.
+`build` runs the generator (`scripts/build.ts`) and then `sync` to propagate
+the JSON and `language-configuration.json` into the app-vscode-ext mirror.
 
 ## Consumers
 
 - **app-promptfiddle** / **pkg-editor** import the JSON directly
   (`import('@b/pkg-grammar/baml.tmLanguage.json')`) — no local copy.
-- **app-vscode-ext** keeps a committed mirror under `syntaxes/` because VS Code
-  loads the grammar from the physical file named in `contributes.grammars[].path`
-  and the extension ships bundled (no `node_modules` in the `.vsix`). The mirror
-  is regenerated by `build` (run automatically on its `dev`/`build` via
-  `sync:grammar`) and a pre-commit hook (`pnpm --filter @b/pkg-grammar check`)
-  fails if it drifts.
+- **app-vscode-ext** keeps committed mirrors (`syntaxes/baml.tmLanguage.json`,
+  `language-configuration.json`) because VS Code loads them from the physical
+  files named in `contributes` and the extension ships bundled (no
+  `node_modules` in the `.vsix`). The mirrors are regenerated by `build` and a
+  pre-commit hook (`pnpm --filter @b/pkg-grammar check`) fails if they drift.
+- **[BoundaryML/textMate-baml](https://github.com/BoundaryML/textMate-baml)**
+  is an external mirror assembled by `scripts/assemble-mirror.mjs` from the
+  templates in `mirror/` and pushed by the `sync-grammar-mirror` workflow on
+  every grammar change on `canary`. It is never edited by hand. It serves
+  three consumers at once:
+  - **npm**: the mirror's own publish workflow releases it as
+    [`@boundaryml/baml-grammar`](https://www.npmjs.com/package/@boundaryml/baml-grammar).
+  - **GitHub Linguist** vendors it as a submodule for `.baml` highlighting on
+    github.com (picked up automatically at each Linguist release, roughly
+    quarterly).
+  - **Shiki's grammar registry** fetches the raw grammar from it weekly.
+
+## Don't break github.com
+
+Linguist compiles every vendored grammar with its own compiler, which converts
+each Oniguruma regex to PCRE — regexes that work in Shiki/VS Code can fail
+there. The `linguist-compile-gate` CI job runs that exact compiler against the
+freshly built grammar, so a grammar change that would break github.com fails
+the PR instead of surfacing months later at a Linguist release. Two invariants
+Linguist pins that must never change: `scopeName: source.baml` and the `.baml`
+file type (guarded in `tests/package-artifacts.test.ts`).
diff --git a/typescript2/pkg-grammar/language-configuration.json b/typescript2/pkg-grammar/language-configuration.json
@@ -0,0 +1,34 @@
+{
+  "comments": {
+    "lineComment": "//",
+    "blockComment": ["{/*", "*/}"]
+  },
+  "brackets": [["{", "}"], ["[", "]"], ["(", ")"]],
+  "autoClosingPairs": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"],
+    { "open": "\"", "close": "\"" },
+    ["#\"", "\"#"],
+    ["'", "'"],
+    ["{#", "#}"],
+    ["{/*", "*/}"]
+  ],
+  "surroundingPairs": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"],
+    ["\"", "\""],
+    ["'", "'"]
+  ],
+  "onEnterRules": [
+    {
+      "beforeText": "^\\s*(?:impl|class|function|enum|prompt).*?:\\s*$",
+      "action": { "indent": "indentOutdent" }
+    },
+    {
+      "beforeText": "#\"",
+      "action": { "indent": "indentOutdent" }
+    }
+  ]
+}